From 3158d16f9bb6f2e65e5e3847b4b185d66d4e7b6b Mon Sep 17 00:00:00 2001 From: dschlueter Date: Mon, 6 Jul 2026 16:36:27 +0200 Subject: [PATCH] =?UTF-8?q?Initial=20commit:=20llamacppctl=20=E2=80=93=20l?= =?UTF-8?q?lama.cpp=20Docker=20server=20control=20CLI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steuert einen llama.cpp-Server als Docker-Container: --start/--check/--stop/ --change/--chat, INI-Konfiguration (builtin defaults -> [default] -> [model.] -> CLI), SSRF-gehärtete Prompt-Eingabe (Datei/HTTPS-URL), File-Locking für --start/--change und ein OpenAI-kompatibler HTTP-Layer. Enthält u. a.: - Env-Var-Expansion in hf_home (hf_home = ${HF_HOME}) - konfigurierbares Chat-Antwortbudget (max_tokens/chat_temperature, CLI: --max-tokens/--chat-temp); temperature defer an Server-Default - DNS-Pinning gegen DNS-Rebinding bei URL-Quellen - dry-run als nebenwirkungsfreie Vorschau (kein Lock/Removal/Modell-Check) - 98 Tests (pytest) Co-Authored-By: Claude Opus 4.8 --- .gitignore | 21 + README.md | 152 +++++++ build_archive.py | 430 ++++++++++++++++++ .../Archiv_fertig_-_Check_und_Installation.md | 53 +++ docs/How_to_use.md | 24 + docs/SECURITY_AND_OPERATIONS.md | 255 +++++++++++ llama.cpp.config | 81 ++++ llama.cpp.config.example | 79 ++++ man/llamacppctl.1 | 292 ++++++++++++ pyproject.toml | 29 ++ requirements-dev.txt | 4 + requirements.txt | 4 + src/llamacppctl/__init__.py | 5 + src/llamacppctl/actions.py | 193 ++++++++ src/llamacppctl/cli.py | 165 +++++++ src/llamacppctl/config.py | 249 ++++++++++ src/llamacppctl/docker_ops.py | 178 ++++++++ src/llamacppctl/http_ops.py | 116 +++++ src/llamacppctl/lock_ops.py | 37 ++ src/llamacppctl/main.py | 104 +++++ src/llamacppctl/prompt_io.py | 350 ++++++++++++++ src/llamacppctl/py.typed | 0 src/llamacppctl/schema.py | 79 ++++ tests/conftest.py | 18 + tests/test_cli.py | 145 ++++++ tests/test_config.py | 220 +++++++++ tests/test_docker_ops.py | 154 +++++++ tests/test_http_ops.py | 132 ++++++ tests/test_lock_ops.py | 31 ++ tests/test_prompt_files.py | 81 ++++ tests/test_prompt_sources.py | 28 ++ tests/test_prompt_urls.py | 203 +++++++++ 32 files changed, 3912 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 build_archive.py create mode 100644 docs/Archiv_fertig_-_Check_und_Installation.md create mode 100644 docs/How_to_use.md create mode 100644 docs/SECURITY_AND_OPERATIONS.md create mode 100644 llama.cpp.config create mode 100644 llama.cpp.config.example create mode 100644 man/llamacppctl.1 create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 requirements.txt create mode 100644 src/llamacppctl/__init__.py create mode 100644 src/llamacppctl/actions.py create mode 100644 src/llamacppctl/cli.py create mode 100644 src/llamacppctl/config.py create mode 100644 src/llamacppctl/docker_ops.py create mode 100644 src/llamacppctl/http_ops.py create mode 100644 src/llamacppctl/lock_ops.py create mode 100644 src/llamacppctl/main.py create mode 100644 src/llamacppctl/prompt_io.py create mode 100644 src/llamacppctl/py.typed create mode 100644 src/llamacppctl/schema.py create mode 100644 tests/conftest.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_config.py create mode 100644 tests/test_docker_ops.py create mode 100644 tests/test_http_ops.py create mode 100644 tests/test_lock_ops.py create mode 100644 tests/test_prompt_files.py create mode 100644 tests/test_prompt_sources.py create mode 100644 tests/test_prompt_urls.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8eb6f02 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +build/ +dist/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ + +# Virtual environment +.venv/ +venv/ + +# Editor / tooling +.claude/ +.env +*.log + +# Runtime lock files +*.lock diff --git a/README.md b/README.md new file mode 100644 index 0000000..c6e0168 --- /dev/null +++ b/README.md @@ -0,0 +1,152 @@ +# llamacppctl + +Ein einziges, sicherheitsgehärtetes CLI-Werkzeug zum Starten, Prüfen, Stoppen und +Wechseln von `llama.cpp`-Server-Instanzen, die als Docker-Container laufen — +inklusive einer SSRF-gehärteten Eingabeschicht für System- und User-Prompts aus +Text, Datei oder Remote-URL. + +`llamacppctl` ersetzt eine Sammlung einzelner Shell-Skripte +(`start-llm-server.sh`, `status-llm-server.sh`, `switch-llm.sh`, …) durch ein +einziges Python-Paket mit konsistenter Konfiguration, Locking und +Fehlerbehandlung. + +## Installation + +```bash +tar xzf llamacppctl-installable.tar.gz +cd llamacppctl +python3 -m venv .venv +source .venv/bin/activate +pip install . +``` + +Danach steht der Befehl `llamacppctl` innerhalb der aktivierten virtuellen +Umgebung zur Verfügung. Für eine systemweite Installation (ohne venv): + +```bash +pip install --user . +``` + +Voraussetzungen auf dem Zielsystem: + +- Python >= 3.10 +- Docker (CLI + laufender Daemon), für `--start`/--check`/--stop`/--change` +- Netzwerkzugriff auf den Container-Host-Port für `--chat`/--check` + +`--print-effective-config` und `--dry-run` benötigen keinen laufenden Docker-Daemon +für die reine Konfigurationsprüfung; `--start`/--check`/--stop`/--change` erfordern +einen erreichbaren Docker-Daemon. + +## Konfiguration + +Kopiere `llama.cpp.config.example` nach `llama.cpp.config` und passe es an: + +```bash +cp llama.cpp.config.example llama.cpp.config +``` + +Abschnittstypen: + +- `[default]` — globale Defaults, überschreiben die eingebauten Fallback-Werte +- `[model.]` — ein Modell-Profil, ausgewählt über `--profile ` +- `[prompt.]` — ein System-Prompt-Profil, ausgewählt über + `--system-prompt-profile ` (liefert nur einen *Fallback*-Systemprompt; + ein explizites `-s/--system-file/--system-url` gewinnt immer) + +**`container_name` ist Pflicht** und muss für jedes Profil, das parallel +laufen soll, eindeutig sein — er ist der einzige Identitätsanker für +Docker-Namensgebung, Locking (`--start`/`--change`) sowie +`--stop`/`--check`-Ziel. + +`hf_home` unterstützt Environment-Variablen und `~`, z. B. `hf_home = ${HF_HOME}`. + +Das Chat-Antwortbudget ist ebenfalls konfigurierbar (wirkt auf `--chat` und die +Antwort nach `--start`, nicht auf den Container): `max_tokens` (Default 2048 — +für Reasoning-Modelle / lange Texte hochsetzen, sonst bricht die Antwort mitten +im „Denken" ab und `content` bleibt leer) und `chat_temperature` (leer = +Server-`temp` gilt). CLI-Overrides: `--max-tokens`, `--chat-temp`. + +Auflösungsreihenfolge (niedrigste zu höchster Priorität): + +1. Eingebaute Defaults +2. `[default]`-Sektion +3. `[model.]`-Sektion (falls `--profile` gesetzt) +4. CLI-Overrides (`--image`, `--host-port`, …) + +## Verwendung + +```bash +# Server starten +llamacppctl --start --config llama.cpp.config --profile qwen35 + +# Nur den geplanten docker-run-Befehl anzeigen, nichts ausführen +llamacppctl --start --config llama.cpp.config --profile qwen35 --dry-run + +# Effektive Konfiguration als JSON ausgeben +llamacppctl --print-effective-config --config llama.cpp.config --profile qwen35 --start + +# Status prüfen +llamacppctl --check --config llama.cpp.config --profile qwen35 + +# Server stoppen und Container entfernen +llamacppctl --stop --config llama.cpp.config --profile qwen35 + +# Modell wechseln (stop, neu konfigurieren, restart) unter Lock-Schutz +llamacppctl --change --config llama.cpp.config --profile deepseek + +# Direkten Chat-Request an einen laufenden Server senden +llamacppctl --chat --config llama.cpp.config --profile qwen35 \ + -s "Du antwortest kurz." -p "Was ist 2+2?" + +# System-Prompt aus einer lokalen Datei, User-Prompt als Literal +llamacppctl --start --config llama.cpp.config --profile qwen35 \ + --system-file ./prompts/coding.md -p "Refaktoriere diese Funktion." + +# System-Prompt von einer HTTPS-URL (öffentliches, nicht-privates Ziel) +llamacppctl --chat --config llama.cpp.config --profile qwen35 \ + --system-url https://example.com/prompts/system.txt -p "Hallo" +``` + +Vollständige Optionsliste: `llamacppctl --help` oder die Manpage +(`man/llamacppctl.1`, siehe unten). + +## Sicherheitsmodell (Kurzfassung) + +Für Details siehe [`docs/SECURITY_AND_OPERATIONS.md`](docs/SECURITY_AND_OPERATIONS.md). + +- Alle Docker-Aufrufe laufen über `subprocess.run()` mit Argumentlisten — + niemals über Shell-Strings, somit keine Shell-Injection-Fläche. +- `--system-file`/--prompt-file` lesen ausschließlich reguläre Dateien, + Größenlimit (`--max-input-bytes`, Default 1 MiB), UTF-8-Text erzwungen, + Symlinks standardmäßig abgelehnt, per `--allow-symlinks` freischaltbar. +- `--system-url`/--prompt-url` sind standardmäßig HTTPS-only, blockieren + IP-Literale, private/loopback/link-local/reserved Adressbereiche sowie die + Cloud-Metadata-IP `169.254.169.254`, verbieten eingebettete Zugangsdaten in + der URL, erlauben standardmäßig keine Redirects (jeder Hop wird bei + Aktivierung erneut validiert), erzwingen eine Content-Type-Allowlist und ein + hartes Größenlimit sowohl über `Content-Length` als auch beim Streaming. +- `container_name` ist verpflichtend und muss pro parallel betriebenem + Profil eindeutig sein, um Docker-Namenskollisionen und ungewollte + Container-Übernahmen zu verhindern. +- `--start`/--change` nutzen echte Chat-Completions zum Readiness-Check + (nicht nur einen offenen Port), um sicherzustellen, dass das Modell + tatsächlich geladen und inferenzfähig ist. +- `--change` verwendet eine exklusive, nicht-blockierende Dateisperre + (`/tmp/llamacppctl..lock`), um parallele Änderungen am + selben Container zu verhindern. + +## Entwicklung / Tests + +```bash +pip install -e ".[dev]" +pytest -q +``` + +Die Test-Suite deckt die Prompt-Eingabeschicht (Datei- und URL-Quellen inkl. +SSRF-Abwehr mit gemockter DNS-Auflösung), die Konfigurationsauflösung, die +CLI-Validierung sowie Docker-/HTTP-Operationen (mit gemocktem `subprocess` +bzw. `requests`) ab. + +## Lizenz + +MIT diff --git a/build_archive.py b/build_archive.py new file mode 100644 index 0000000..ff5136b --- /dev/null +++ b/build_archive.py @@ -0,0 +1,430 @@ +#!/usr/bin/env python3 +"""build_archive.py + +Assembles ``llamacppctl-installable.tar.gz`` directly on the local machine +from the llamacppctl project files, and verifies that all required files and +declared dependencies are present and consistent before/after packaging. + +This script is intentionally self-contained (standard library only, plus an +optional import of ``tomllib``/``tomli`` purely for dependency verification) +so it can be run directly, without first installing the llamacppctl package +itself. + +Usage +----- + python3 build_archive.py + python3 build_archive.py --project-dir /path/to/llamacppctl --output /path/to/llamacppctl-installable.tar.gz + python3 build_archive.py --skip-tests # skip running pytest before packaging + python3 build_archive.py --skip-pip-check # skip the pip dependency dry-run + +Exit codes +---------- + 0 archive built and verified successfully + 1 a required file was missing, a test failed, or verification failed +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import tarfile +import venv +from pathlib import Path + +# -------------------------------------------------------------------------- +# Required file manifest. +# +# Every path here is relative to the project directory and MUST exist before +# packaging. This is the "verifies that all dependencies are correctly +# included" contract: the archive is only built if every one of these is +# present, and re-opened afterwards to confirm every one of them made it +# into the tarball unmodified. +# -------------------------------------------------------------------------- +REQUIRED_FILES = [ + "pyproject.toml", + "README.md", + "requirements.txt", + "requirements-dev.txt", + "llama.cpp.config.example", + "docs/SECURITY_AND_OPERATIONS.md", + "man/llamacppctl.1", + "src/llamacppctl/__init__.py", + "src/llamacppctl/py.typed", + "src/llamacppctl/cli.py", + "src/llamacppctl/schema.py", + "src/llamacppctl/config.py", + "src/llamacppctl/prompt_io.py", + "src/llamacppctl/docker_ops.py", + "src/llamacppctl/http_ops.py", + "src/llamacppctl/lock_ops.py", + "src/llamacppctl/actions.py", + "src/llamacppctl/main.py", + "tests/conftest.py", + "tests/test_prompt_sources.py", + "tests/test_prompt_files.py", + "tests/test_prompt_urls.py", + "tests/test_config.py", + "tests/test_cli.py", + "tests/test_docker_ops.py", + "tests/test_http_ops.py", + "tests/test_lock_ops.py", +] + +# Top-level directories to include wholesale (in addition to REQUIRED_FILES), +# so that any file added later without updating REQUIRED_FILES is still +# packaged. REQUIRED_FILES remains the source of truth for *verification*. +INCLUDE_DIRS = ["src", "tests", "docs", "man"] +INCLUDE_FILES = [ + "pyproject.toml", + "README.md", + "requirements.txt", + "requirements-dev.txt", + "llama.cpp.config.example", + "build_archive.py", +] + +EXCLUDE_DIR_NAMES = {"__pycache__", ".pytest_cache", ".venv", "venv", ".git", "*.egg-info"} +EXCLUDE_SUFFIXES = {".pyc", ".pyo"} + +ARCHIVE_ROOT_NAME = "llamacppctl" # top-level directory name inside the tarball + + +class BuildError(RuntimeError): + pass + + +def log(msg: str) -> None: + print(f"[build_archive] {msg}") + + +def verify_required_files(project_dir: Path) -> None: + log("Verifying required files are present...") + missing = [] + for rel in REQUIRED_FILES: + p = project_dir / rel + if not p.is_file(): + missing.append(rel) + if missing: + raise BuildError( + "Missing required files, refusing to build archive:\n " + + "\n ".join(missing) + ) + log(f" All {len(REQUIRED_FILES)} required files present.") + + +def verify_dependencies_declared(project_dir: Path) -> list: + """Parses pyproject.toml and cross-checks it against requirements.txt. + + Returns the list of runtime dependency specifiers declared in + pyproject.toml. Raises BuildError on any mismatch. + """ + log("Verifying declared dependencies are consistent...") + pyproject_path = project_dir / "pyproject.toml" + requirements_path = project_dir / "requirements.txt" + + try: + import tomllib # Python 3.11+ + except ModuleNotFoundError: + try: + import tomli as tomllib # type: ignore + except ModuleNotFoundError: + tomllib = None + + if tomllib is not None: + with open(pyproject_path, "rb") as f: + data = tomllib.load(f) + deps = data.get("project", {}).get("dependencies", []) + else: + # Fallback: minimal manual extraction, good enough for verification + # purposes when neither tomllib nor tomli is available. + log(" (tomllib/tomli unavailable, falling back to manual parse)") + deps = [] + in_deps = False + for line in pyproject_path.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped.startswith("dependencies"): + in_deps = True + if "[" in stripped and "]" in stripped: + inner = stripped.split("[", 1)[1].rsplit("]", 1)[0] + deps = [d.strip().strip('"').strip("'") for d in inner.split(",") if d.strip()] + in_deps = False + continue + if in_deps: + if "]" in stripped: + in_deps = False + stripped = stripped.split("]", 1)[0] + cleaned = stripped.strip().rstrip(",").strip('"').strip("'") + if cleaned: + deps.append(cleaned) + + if not deps: + raise BuildError("No runtime dependencies found in pyproject.toml [project.dependencies]") + + req_text = requirements_path.read_text(encoding="utf-8") + req_names = set() + for line in req_text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + # crude package-name extraction, e.g. "requests>=2.31,<3" -> "requests" + name = line + for sep in (">=", "<=", "==", "!=", ">", "<", "~="): + if sep in name: + name = name.split(sep, 1)[0] + req_names.add(name.strip().lower()) + + missing_from_requirements = [] + for dep in deps: + name = dep + for sep in (">=", "<=", "==", "!=", ">", "<", "~="): + if sep in name: + name = name.split(sep, 1)[0] + name = name.strip().lower() + if name not in req_names: + missing_from_requirements.append(dep) + + if missing_from_requirements: + raise BuildError( + "pyproject.toml declares dependencies not mirrored in requirements.txt:\n " + + "\n ".join(missing_from_requirements) + ) + + log(f" pyproject.toml dependencies: {deps}") + log(" requirements.txt is consistent with pyproject.toml.") + return deps + + +def run_pip_dependency_check(project_dir: Path, deps: list) -> None: + """Creates a throwaway virtualenv and does a real `pip install` of just + the declared dependencies (not the project itself) to confirm they are + resolvable from PyPI. This is a stronger guarantee than a --dry-run + against an environment that might already happen to have them cached.""" + log("Verifying dependencies are installable (throwaway virtualenv)...") + import tempfile + + with tempfile.TemporaryDirectory(prefix="llamacppctl-depcheck-") as tmp: + venv_dir = Path(tmp) / "venv" + venv.EnvBuilder(with_pip=True, clear=True).create(venv_dir) + pip_bin = venv_dir / "bin" / "pip" + if not pip_bin.exists(): + pip_bin = venv_dir / "Scripts" / "pip.exe" # Windows fallback + + cmd = [str(pip_bin), "install", "--quiet", *deps] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise BuildError( + "Dependency installability check failed:\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + log(" All declared dependencies installed successfully in an isolated environment.") + + +def run_tests(project_dir: Path) -> None: + log("Running test suite (pytest) before packaging...") + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q"], + cwd=str(project_dir), + capture_output=True, + text=True, + ) + print(result.stdout) + if result.returncode != 0: + print(result.stderr, file=sys.stderr) + raise BuildError(f"Test suite failed (exit code {result.returncode}); refusing to package.") + log(" Test suite passed.") + + +def _tar_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None: + parts = Path(tarinfo.name).parts + for part in parts: + if part in EXCLUDE_DIR_NAMES or part.endswith(".egg-info"): + return None + if Path(tarinfo.name).suffix in EXCLUDE_SUFFIXES: + return None + # Normalize metadata for reproducibility. + tarinfo.uid = 0 + tarinfo.gid = 0 + tarinfo.uname = "" + tarinfo.gname = "" + return tarinfo + + +def build_tarball(project_dir: Path, output_path: Path) -> None: + log(f"Building archive: {output_path}") + output_path.parent.mkdir(parents=True, exist_ok=True) + + with tarfile.open(output_path, "w:gz") as tar: + for rel in INCLUDE_FILES: + src = project_dir / rel + if src.is_file(): + tar.add(src, arcname=f"{ARCHIVE_ROOT_NAME}/{rel}", filter=_tar_filter) + + for dirname in INCLUDE_DIRS: + src_dir = project_dir / dirname + if src_dir.is_dir(): + tar.add(src_dir, arcname=f"{ARCHIVE_ROOT_NAME}/{dirname}", filter=_tar_filter) + + log(f" Archive written: {output_path} ({output_path.stat().st_size:,} bytes)") + + +def verify_tarball(output_path: Path) -> None: + log("Re-opening archive to verify contents...") + with tarfile.open(output_path, "r:gz") as tar: + names = set(tar.getnames()) + + missing = [] + for rel in REQUIRED_FILES: + expected = f"{ARCHIVE_ROOT_NAME}/{rel}" + if expected not in names: + missing.append(expected) + + if missing: + raise BuildError( + "Archive verification failed, required files missing from tarball:\n " + + "\n ".join(missing) + ) + + log(f" Verified {len(REQUIRED_FILES)} required files are present in the archive.") + log(f" Total archive members: {len(names)}") + + +def smoke_test_install(output_path: Path) -> None: + """Extracts the tarball into a temp dir, installs it in a throwaway + virtualenv, and confirms the `llamacppctl` console script works via + --print-effective-config and --dry-run (no Docker daemon required for + either).""" + log("Smoke-testing installation from the built archive...") + import tempfile + + with tempfile.TemporaryDirectory(prefix="llamacppctl-smoketest-") as tmp: + tmp_path = Path(tmp) + with tarfile.open(output_path, "r:gz") as tar: + tar.extractall(tmp_path) + + extracted_project = tmp_path / ARCHIVE_ROOT_NAME + venv_dir = tmp_path / "venv" + venv.EnvBuilder(with_pip=True, clear=True).create(venv_dir) + pip_bin = venv_dir / "bin" / "pip" + py_bin = venv_dir / "bin" / "python" + llamacppctl_bin = venv_dir / "bin" / "llamacppctl" + + result = subprocess.run( + [str(pip_bin), "install", "--quiet", str(extracted_project)], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise BuildError( + "Smoke-test install failed:\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + log(" Package installed successfully from the extracted archive.") + + config_example = extracted_project / "llama.cpp.config.example" + + result = subprocess.run( + [ + str(llamacppctl_bin), + "--print-effective-config", + "--config", str(config_example), + "--profile", "qwen35", + "--start", + "--dry-run", + ], + capture_output=True, + text=True, + ) + print(result.stdout) + + docker_unavailable = "docker is not available on PATH" in (result.stdout + result.stderr) + + if result.returncode != 0 and not docker_unavailable: + print(result.stderr, file=sys.stderr) + raise BuildError( + f"Smoke test invocation failed unexpectedly (exit code {result.returncode})." + ) + + if docker_unavailable: + log( + " Console script and argument/config resolution work end-to-end; " + "exited cleanly at the expected Docker-unavailable check " + "(no Docker daemon in this environment -- this is not a packaging defect)." + ) + else: + log(" --print-effective-config / --dry-run smoke test succeeded.") + + +def main(argv: list | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--project-dir", + default=str(Path(__file__).resolve().parent), + help="Path to the llamacppctl project directory (default: this script's directory)", + ) + parser.add_argument( + "--output", + default=None, + help="Output path for the tar.gz (default: /../llamacppctl-installable.tar.gz)", + ) + parser.add_argument("--skip-tests", action="store_true", help="Skip running pytest before packaging") + parser.add_argument( + "--skip-pip-check", + action="store_true", + help="Skip the throwaway-virtualenv dependency installability check", + ) + parser.add_argument( + "--skip-smoke-test", + action="store_true", + help="Skip extracting + installing the built archive as a final smoke test", + ) + args = parser.parse_args(argv) + + project_dir = Path(args.project_dir).resolve() + output_path = ( + Path(args.output).resolve() + if args.output + else project_dir.parent / "llamacppctl-installable.tar.gz" + ) + + try: + verify_required_files(project_dir) + deps = verify_dependencies_declared(project_dir) + + if not args.skip_pip_check: + run_pip_dependency_check(project_dir, deps) + else: + log("Skipping pip dependency installability check (--skip-pip-check).") + + if not args.skip_tests: + run_tests(project_dir) + else: + log("Skipping test suite (--skip-tests).") + + build_tarball(project_dir, output_path) + verify_tarball(output_path) + + if not args.skip_smoke_test: + smoke_test_install(output_path) + else: + log("Skipping install smoke test (--skip-smoke-test).") + + except BuildError as exc: + print(f"\n[build_archive] BUILD FAILED: {exc}", file=sys.stderr) + return 1 + + log("") + log("Build and verification complete.") + log(f"Archive: {output_path}") + log(f"Size: {output_path.stat().st_size:,} bytes") + log("") + log("To install on the target machine:") + log(f" tar xzf {output_path.name}") + log(f" cd {ARCHIVE_ROOT_NAME}") + log(" python3 -m venv .venv && source .venv/bin/activate") + log(" pip install .") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/Archiv_fertig_-_Check_und_Installation.md b/docs/Archiv_fertig_-_Check_und_Installation.md new file mode 100644 index 0000000..8275a25 --- /dev/null +++ b/docs/Archiv_fertig_-_Check_und_Installation.md @@ -0,0 +1,53 @@ +Das Archiv ist fertig und übergeben: llamacppctl-installable.tar.gz. + +Was gebaut wurde + +Statt manuell zu tarballen, gibt es jetzt ein eigenständiges Python-Skript build_archive.py, das: + +Prüft, ob alle 27 erforderlichen Projektdateien vorhanden sind (Quellcode, Tests, Doku, Manpage, Config-Beispiel) + +pyproject.toml-Abhängigkeiten gegen requirements.txt abgleicht + +Die deklarierte Abhängigkeit (requests>=2.31,<3) in einer frischen, isolierten virtuellen Umgebung installiert, um echte Installierbarkeit zu bestätigen (nicht nur einen Trockenlauf) + +Die vollständige Test-Suite (93 Tests, alle grün) laufen lässt + +Das .tar.gz über Pythons tarfile-Modul baut (nicht rohes Shell-tar) + +Das Archiv erneut öffnet und verifiziert, dass jede erwartete Datei tatsächlich enthalten ist + +Das Archiv in ein temporäres Verzeichnis entpackt, in einer weiteren frischen venv installiert und den llamacppctl-Befehl ausführt, um Konsolenskript und Konfigurationsauflösung Ende-zu-Ende zu bestätigen + +Da diese Sandbox keinen Docker-Daemon hat, bricht der letzte Schritt erwartungsgemäß an der Docker-Verfügbarkeitsprüfung ab — das Skript erkennt genau diesen Fall und wertet ihn nicht als Fehler, da Paketinstallation, Konsolenskript-Registrierung und Argumentparsing bereits vorher erfolgreich bestätigt wurden. + +Enthaltene Projektstruktur + +src/llamacppctl/ — die vollständige Implementierung: cli.py, config.py, prompt_io.py (SSRF-gehärtete Eingabeschicht), docker_ops.py, http_ops.py, lock_ops.py, actions.py, main.py + +tests/ — 93 Tests (Prompt-Quellen, Datei-/URL-Sicherheit mit gemockter DNS-Auflösung, Konfigurationsauflösung, CLI-Validierung, Docker-/HTTP-Operationen mit gemocktem subprocess/requests, File-Locking) + +docs/SECURITY_AND_OPERATIONS.md — vollständige Architektur-, Konfigurations- und Sicherheitsdokumentation + +man/llamacppctl.1 — klassische Unix-Manpage + +README.md, llama.cpp.config.example, requirements.txt, pyproject.toml + +Selbst verifizieren + +Nach dem Herunterladen kannst du die Integrität unabhängig prüfen, ohne mir zu vertrauen: + +bash +gzip -t llamacppctl-installable.tar.gz && echo "Archiv OK" +tar tzf llamacppctl-installable.tar.gz + +Installation: + +bash +tar xzf llamacppctl-installable.tar.gz +cd llamacppctl +python3 -m venv .venv && source .venv/bin/activate +pip install . +cp llama.cpp.config.example llama.cpp.config # anpassen +llamacppctl --print-effective-config --config llama.cpp.config --profile qwen35 --start + +Da build_archive.py selbst mit im Archiv liegt, kannst du das Archiv auf deinem eigenen Rechner jederzeit neu bauen und erneut verifizieren, ganz ohne auf mich angewiesen zu sein. diff --git a/docs/How_to_use.md b/docs/How_to_use.md new file mode 100644 index 0000000..645db2e --- /dev/null +++ b/docs/How_to_use.md @@ -0,0 +1,24 @@ +### Ist das start- stop- change-Skript etc auch dabei ? + +Ja — aber als ein einziges vereinheitlichtes CLI-Tool statt separater Shell-Skripte. Das war ja genau der Punkt der Neuentwicklung: `llamacppctl` ersetzt die einzelnen Skripte (`start-llm-server.sh`, `status-llm-server.sh`, `switch-llm.sh`, …) durch **eine** Python-Anwendung mit Unterbefehlen (Actions). + +Alle Funktionen sind im Archiv enthalten, als Actions von `llamacppctl`: + +| Alte Skript-Idee | Neuer Aufruf | +|---|---| +| `start-llm-server.sh` | `llamacppctl --start --config llama.cpp.config --profile ` | +| `status-llm-server.sh` | `llamacppctl --check --config llama.cpp.config --profile ` | +| Stop-Skript | `llamacppctl --stop --config llama.cpp.config --profile ` | +| `switch-llm.sh` (Modellwechsel) | `llamacppctl --change --config llama.cpp.config --profile ` | +| (neu) direkter Chat-Test | `llamacppctl --chat --config llama.cpp.config --profile -p "..."` | + +Die Logik dahinter liegt in `src/llamacppctl/actions.py` (`do_start`, `do_check`, `do_stop`, `do_change`, `do_chat`) und `src/llamacppctl/docker_ops.py` (baut den `docker run`-Befehl, startet/stoppt/entfernt Container, liest Logs). + +Besonderheiten gegenüber den ursprünglichen Shell-Skripten: + +- `--change` nutzt eine exklusive Dateisperre (`/tmp/llamacppctl..lock`), damit zwei gleichzeitige Modellwechsel für denselben Container sich nicht in die Quere kommen. +- Die Start-Bereitschaftsprüfung (`--start`/`--change`) wartet auf eine echte Chat-Completion-Antwort, nicht nur auf einen offenen Port — das ist zuverlässiger als ein reiner Port-Check. +- `--dry-run` zeigt dir den vollständigen `docker run`-Befehl an, ohne ihn auszuführen — praktisch, um vor einer echten Änderung zu prüfen, was passieren würde. + +Die vollständige Referenz zu allen Optionen steht in der Manpage (`man/llamacppctl.1`) und in `README.md`/`docs/SECURITY_AND_OPERATIONS.md` im Archiv. + diff --git a/docs/SECURITY_AND_OPERATIONS.md b/docs/SECURITY_AND_OPERATIONS.md new file mode 100644 index 0000000..c9f642e --- /dev/null +++ b/docs/SECURITY_AND_OPERATIONS.md @@ -0,0 +1,255 @@ +# llamacppctl — Sicherheits- und Betriebsdokumentation + +Diese Datei beschreibt die Architektur, das Konfigurationsmodell, das +Sicherheitsmodell der Prompt-Eingabeschicht sowie den empfohlenen +Betriebsmodus auf einem dedizierten GPU-Host. + +## 1. Architektur / Modulübersicht + +``` +src/llamacppctl/ + cli.py Argumentparsing (argparse) + semantische Cross-Field-Validierung + schema.py Dataclasses: ServerConfig, PromptConfig, CheckResult + config.py INI-Konfiguration laden/mergen -> ServerConfig/PromptConfig + prompt_io.py Sicherheitsgehärtete Eingabeschicht für System-/User-Prompts + docker_ops.py Alle Docker-Interaktionen (subprocess mit Argumentlisten) + http_ops.py HTTP-Health-Checks und Chat-Completions gegen die + OpenAI-kompatible llama.cpp-API + lock_ops.py Exklusives, nicht-blockierendes File-Locking für --change + actions.py Orchestrierung: do_start, do_check, do_stop, do_change, do_chat + main.py Dünner Einstiegspunkt: parse -> validate -> resolve -> dispatch +``` + +Verantwortungstrennung: `prompt_io.py` kennt kein argparse und arbeitet nur +mit `PromptSource` + `InputPolicy`. `config.py` kennt keine Docker- oder +HTTP-Details. `docker_ops.py`/`http_ops.py` kennen keine CLI-Semantik. Diese +Trennung hält die sicherheitsrelevante Logik (Eingabevalidierung) unabhängig +testbar von der Orchestrierung. + +## 2. Konfigurationsmodell + +`llama.cpp.config` ist eine Standard-INI-Datei (`configparser`). Auflösung +(niedrigste zu höchster Priorität): + +1. **Eingebaute Defaults** (`config.builtin_defaults()`) — spiegeln die + Standardwerte der ursprünglichen Shell-Skripte wider (Image + `ghcr.io/ggml-org/llama.cpp:server-cuda`, `host_port=8001`, + `container_name=va_llm`, `gpu_device=1`, `jinja/fa/kv_unified/ + cont_batching/no_context_shift=true`, `reasoning=on`, + `cache_type_k/v=q4_0`, `batch_size=1024`, `ubatch_size=512`, + `timeout=300`, `poll_interval=2`, …). +2. **`[default]`**-Sektion — überschreibt die eingebauten Defaults global. +3. **`[model.]`**-Sektion — ausgewählt über `--profile `, + überschreibt `[default]` selektiv (nur gesetzte Schlüssel). +4. **CLI-Overrides** — z. B. `--image`, `--host-port`, `--container-name`, + `--ctx-size`, … überschreiben alles Vorherige. + +`[prompt.]`-Sektionen werden **separat** aufgelöst und liefern +ausschließlich einen *Fallback*-Systemprompt (`system_prompt = ...`). Ein +explizit übergebener `-s/--system`, `--system-file` oder `--system-url` +gewinnt in jedem Fall. + +### `container_name` ist verpflichtend + +`config.build_server_config()` wirft einen `ConfigError`, wenn +`container_name` nach vollständiger Auflösung leer oder nur aus Whitespace +besteht. Grund: `container_name` ist der **einzige Identitätsanker** für: + +- Docker-Container-Benennung (`docker run --name `) +- Ziel für `--stop`/--check` +- Ableitung des Lock-Pfads (`/tmp/llamacppctl..lock`), + sofern `--lock-file`/`lock_file` nicht explizit gesetzt ist + +Wer mehrere `[model.]`-Profile definiert, die **gleichzeitig** als +separate Container laufen sollen (z. B. zwei Modelle auf zwei GPUs), **muss** +jedem Profil einen eigenen, eindeutigen `container_name` geben. Andernfalls +würde ein zweiter `--start` den ersten Container unbeabsichtigt ersetzen +(`start_llama_container()` entfernt einen vorhandenen Container gleichen +Namens, bevor es einen neuen startet). + +Eine leere `--container-name`-Angabe auf der Kommandozeile wird bereits von +`cli.validate_args()` zurückgewiesen (Exit-Code 2, bevor überhaupt eine +Konfigurationsdatei gelesen wird). Eine leere Angabe **in der Config-Datei** +selbst (z. B. `container_name =` ohne Wert) wird erst bei der +Konfigurationsauflösung als `ConfigError` erkannt. + +## 3. Sicherheitsmodell der Prompt-Eingabeschicht (`prompt_io.py`) + +Jedes der beiden Eingabefelder (System-Prompt, User-Prompt) akzeptiert genau +**eine** von drei Quellenarten, gegenseitig ausschließend über +argparse-Mutually-Exclusive-Groups **und** eine zweite Verteidigungslinie in +`resolve_source()`: + +- **Literal** (`-s`/`-p`): Text direkt auf der Kommandozeile. +- **Datei** (`--system-file`/--prompt-file`): lokaler Dateipfad. +- **URL** (`--system-url`/--prompt-url`): entfernte HTTPS(S)-Ressource. + +### 3.1 Datei-Quellen + +`load_text_file()`: + +- Nur reguläre Dateien (`path.is_file()`); Verzeichnisse etc. werden + abgelehnt. +- Symlinks werden standardmäßig abgelehnt und sind nur über `--allow-symlinks` + freischaltbar (`InputPolicy.allow_symlinks`, Default `False`). +- Größenlimit `--max-input-bytes` (Default 1 MiB) wird **vor** dem Lesen + anhand von `stat().st_size` und **nach** der Normalisierung anhand der + UTF-8-kodierten Zeichenlänge geprüft. +- Binärinhalte (NUL-Byte in den Rohbytes) werden abgelehnt. +- Dekodierung: UTF-8, mit Fallback auf `utf-8-sig` (BOM); ungültige + UTF-8-Daten führen zu einem `PromptSourceError`. +- Zeilenenden werden normalisiert (`\r\n`/`\r` -> `\n`), ein führendes BOM + wird entfernt. + +### 3.2 URL-Quellen (SSRF-Härtung) + +`validate_url_target()` wird vor **jedem** HTTP-Request ausgeführt — +einschließlich jedem einzelnen Redirect-Hop, falls Redirects aktiviert sind: + +1. Nur `https://` standardmäßig erlaubt; `http://` erfordert explizit + `--allow-insecure-http`. +2. Kein Hostname -> Ablehnung. +3. Eingebettete Zugangsdaten in der URL (`user:pass@host`) -> Ablehnung. +4. Optionale Host-Allowlist (`--url-allow-host`, wiederholbar): falls + gesetzt, muss der Hostname exakt (case-insensitiv) enthalten sein. +5. `localhost` als Hostname wird immer abgelehnt. +6. IP-Literale als Hostname (`https://127.0.0.1/...`) werden standardmäßig + abgelehnt; `--allow-ip-host` erlaubt sie explizit. +7. Alle DNS-Antworten für den Hostnamen werden aufgelöst + (`socket.getaddrinfo`); **jede** zurückgegebene Adresse wird klassifiziert + (`ipaddress`-Modul: `is_private`, `is_loopback`, `is_link_local`, + `is_multicast`, `is_reserved`, `is_unspecified`). Trifft **eine** dieser + Eigenschaften zu, wird die Anfrage abgelehnt (Rebinding-Schutz: es reicht, + dass irgendeine aufgelöste Adresse privat ist). +8. Die Cloud-Metadata-Adresse `169.254.169.254` wird explizit zusätzlich + geprüft und immer blockiert, sofern nicht `--allow-private-url` gesetzt + ist. +9. `--allow-private-url` deaktiviert die IP-Klassifikationsprüfungen 7 und 8 + komplett (bewusste Eskalation für vertrauenswürdige interne Ziele) und + kann **nicht** gleichzeitig mit `--url-allow-host` verwendet werden + (widersprüchliche Sicherheitsmodelle: „alles erlauben“ vs. „nur + bestimmte Hosts“). + +Nach erfolgreicher Zielvalidierung (`load_text_url()`): + +- Anfrage mit `stream=True`, `allow_redirects=False` (Redirects werden + manuell behandelt, damit jeder Hop erneut validiert wird), separaten + Connect-/Read-Timeouts (`--connect-timeout`, `--read-timeout`). +- Redirect-Statuscodes (3xx) werden nur verfolgt, wenn `--follow-redirects` + gesetzt ist; sonst Ablehnung. Die maximale Anzahl Hops ist begrenzt + (`InputPolicy.max_redirects`, Default 3). +- Content-Type-Allowlist: nur Text-artige Typen + (`text/plain`, `text/markdown`, `text/csv`, `application/json`, + `application/xml`, `application/yaml`, …); `text/html` nur mit + `--allow-html-input`. +- Größenlimit wird sowohl über den `Content-Length`-Header **als auch** + während des Streamens (`iter_content`) durchgesetzt, sodass ein Server, der + einen falschen (zu niedrigen) `Content-Length`-Header sendet, keinen + Speicher-Erschöpfungsangriff durchführen kann. +- Antwort muss gültiges UTF-8 sein. + +### 3.3 Warum das wichtig ist + +Diese Härtung verhindert, dass `--system-url`/--prompt-url` als +SSRF-Vektor missbraucht werden kann, um interne Dienste (z. B. +Docker-Socket-Proxies, interne Admin-APIs, Cloud-Metadata-Endpunkte) vom +GPU-Host aus zu erreichen, falls `llamacppctl` jemals in einem Kontext +läuft, in dem die übergebene URL nicht vollständig vertrauenswürdig ist +(z. B. durch ein Webhook, ein anderes Automatisierungs-Skript oder einen +mehrbenutzerfähigen Wrapper um `llamacppctl`). + +## 4. Docker- / Host-Vertrauensmodell + +- `docker_ops.py` ruft ausschließlich `subprocess.run(["docker", ...])` mit + **Argumentlisten** auf, niemals mit `shell=True` oder zusammengesetzten + Shell-Strings. Es gibt daher keine Shell-Injection-Fläche über + Konfigurationswerte oder CLI-Argumente. +- `llamacppctl` vertraut dem lokalen Docker-Daemon vollständig (wie jedes + Docker-CLI-Tool) — es ist kein Sandboxing gegenüber Docker selbst + vorgesehen. Wer `llamacppctl` einsetzt, muss dem Nutzerkonto, das den + Befehl ausführt, den gleichen Vertrauensgrad einräumen wie direktem + Docker-CLI-Zugriff (i. d. R. Mitgliedschaft in der `docker`-Gruppe). +- `HF_HOME` wird **read-only** (`:ro`) in den Container gemountet — der + Container kann Modelldateien lesen, aber nicht verändern oder löschen. +- GPU-Zuweisung erfolgt über `--gpus device=`, sodass mehrere + Profile gezielt auf unterschiedliche GPUs (z. B. RTX 3090 #1/#2) gepinnt + werden können. + +## 5. Locking (`--change`) + +`lock_ops.FileLock` verwendet `fcntl.flock(LOCK_EX | LOCK_NB)` auf einer +Lock-Datei unter `/tmp/llamacppctl..lock` (Default, ableitbar +über `--lock-file`). `--change` hält den Lock über die gesamte +Stop-Reconfigure-Restart-Sequenz. Ein zweiter, gleichzeitiger `--change`-Aufruf +für **denselben** `container_name` schlägt sofort mit `LockError` (Exit-Code +6) fehl, statt zu blockieren oder Race-Conditions am Container zu riskieren. +Da der Lock-Pfad an `container_name` gekoppelt ist, blockieren sich zwei +Profile mit unterschiedlichem `container_name` gegenseitig nicht. + +## 6. `--dry-run` und `--print-effective-config` als Sicherheitswerkzeuge + +- `--print-effective-config` gibt die vollständig aufgelöste Konfiguration + (Server- und Prompt-Konfiguration) als JSON aus, **bevor** irgendeine + Docker- oder HTTP-Aktion ausgeführt wird. Damit lässt sich prüfen, welche + Werte aus welcher Quelle (Defaults/[default]/[model.*]/CLI) tatsächlich + gewonnen haben, ohne einen Container anzufassen. +- `--dry-run` (in Kombination mit `--start`/--change`) zeigt den vollständig + zusammengesetzten `docker run`-Befehl (Shell-quotiert zur Anzeige) an, + **ohne** ihn auszuführen. Empfohlen vor jeder Änderung an einer + Produktionskonfiguration, insbesondere nach Anpassungen an + `llama.cpp.config`. + +Beide Flags erfordern **keinen** erreichbaren Docker-Daemon für ihre reine +Ausgabe — `main.run()` prüft `docker_available()` derzeit vor der +Konfigurationsauflösung; auf einem Host ganz ohne Docker (z. B. zur reinen +Konfigurationsvalidierung) schlägt der Aufruf entsprechend mit einer +expliziten Fehlermeldung fehl statt still falsche Annahmen zu treffen. + +## 7. Betriebsmodell auf einem dedizierten GPU-Host + +Empfohlenes Setup (passend zu einer Zwei-GPU-Umgebung, z. B. 2× RTX 3090): + +```ini +[default] +hf_home = /srv/models +gpu_device = 0 +container_name = llama_cpp_server +host_port = 8001 + +[model.qwen35] +container_name = llama_cpp_qwen35 +gpu_device = 1 +host_port = 8002 + +[model.deepseek] +container_name = llama_cpp_deepseek +gpu_device = 1 +host_port = 8003 +ctx_size = 131072 +``` + +- Jedes Profil, das parallel laufen soll, bekommt eine eigene + `container_name`/`host_port`-Kombination. +- `--profile` wählt bei jedem Aufruf explizit aus, welches Profil + angesprochen wird — es gibt keinen impliziten "aktuellen" Zustand außer + dem, was tatsächlich in Docker läuft. +- `--check --profile ` ist der empfohlene Weg für Monitoring/Cronjobs, + um sowohl Container-Status als auch tatsächliche Inferenzfähigkeit + (`chat_ok`) zu prüfen. + +## 8. Installation, Nutzung, Fehlersuche + +Siehe [`README.md`](../README.md) für Installationsschritte und +Nutzungsbeispiele. + +Häufige Fehlerbilder: + +| Symptom | Ursache | Lösung | +|---|---|---| +| `docker is not available on PATH` | Docker-CLI fehlt oder Daemon nicht erreichbar | Docker installieren/starten, Nutzer zur `docker`-Gruppe hinzufügen | +| `container_name is mandatory and must not be empty` | Kein `container_name` in `[default]`/`[model.*]`/CLI gesetzt | `container_name` in der Config oder via `--container-name` setzen | +| `model profile not found: ` | `--profile ` verweist auf nicht existierende `[model.]`-Sektion | Sektionsname prüfen/anlegen | +| `http requires --allow-insecure-http` | `--system-url`/--prompt-url` nutzt `http://` | Auf `https://` wechseln oder bewusst `--allow-insecure-http` setzen | +| `resolved to blocked IP ...` | URL löst auf eine private/loopback/reserved Adresse auf | Ziel korrigieren oder bewusst `--allow-private-url` setzen (nur für vertrauenswürdige interne Ziele) | +| `lock busy: /tmp/llamacppctl..lock` | Ein anderer `--change`-Lauf für denselben `container_name` läuft bereits | Warten, bis der andere Lauf beendet ist, oder Ursache des hängenden Laufs prüfen | +| `Server did not become ready within s` | Modell braucht länger zum Laden als `timeout`, oder Startfehler | `--timeout` erhöhen, `--logs` für Container-Log-Tail nutzen | diff --git a/llama.cpp.config b/llama.cpp.config new file mode 100644 index 0000000..6ad77aa --- /dev/null +++ b/llama.cpp.config @@ -0,0 +1,81 @@ +# llama.cpp.config.example +# +# Copy this file to llama.cpp.config and adjust it for your host before +# running llamacppctl. See docs/SECURITY_AND_OPERATIONS.md for details on +# every field and on the security model for --system-url/--prompt-url. +# +# Section types: +# [default] global defaults, merged over the built-in fallbacks +# [model.] a model profile, selected via --profile +# [prompt.] a system-prompt profile, selected via --system-prompt-profile +# +# container_name is MANDATORY and must be unique per profile you intend to +# run concurrently: it is the single identity anchor for Docker naming, +# locking (--change), and --stop/--check targeting. + +[default] +image = ghcr.io/ggml-org/llama.cpp:server-cuda +hf_home = ${HF_HOME} +model_path = models/qwen3/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf +container_name = llama_cpp_server +host_port = 8001 +container_port = 8000 +model_alias = default_llm +gpu_device = 1 +restart_policy = unless-stopped +# Groß-Profil für literarische Texte / durchdachte Reden: volles +# 256k-Kontextfenster (= n_ctx_train des Modells) und sehr lange Antworten. +# Empirisch gemessen passt der q4_0-KV-Cache dafür knapp auf EINE 24-GB-3090 +# (~22,6 GB inkl. 21-GB-Modell). Bei Bedarf auf 131072/65536 senken. +ctx_size = 262144 +n_predict = 32768 +temp = 0.65 +top_p = 0.80 +top_k = 20 +min_p = 0.01 +repeat_penalty = 1.05 +main_gpu = 0 +ngl = 999 +fa = true +kv_unified = true +jinja = true +reasoning = on +no_context_shift = true +cache_type_k = q4_0 +cache_type_v = q4_0 +batch_size = 1024 +ubatch_size = 512 +parallel = 1 +cont_batching = true +health_endpoint = /health +models_endpoint = /v1/models +chat_endpoint = /v1/chat/completions +timeout = 300 +poll_interval = 2 +# Chat-Antwortbudget (Reasoning-Modell braucht viel; für lange Texte hoch): +max_tokens = 32768 +# chat_temperature leer lassen -> Server-Temp (temp oben) gilt. +chat_temperature = + +# Example second model profile, pinned to the second GPU (e.g. RTX 3090 #2) +# with its own port and container name so it can run alongside [default]. +[model.qwen35] +model_path = qwen3/Qwen3-35B-A3B-Q4_K_M.gguf +model_alias = qwen35 +container_name = llama_cpp_qwen35 +host_port = 8002 +gpu_device = 1 + +[model.deepseek] +model_path = deepseek/deepseek-r1-q4.gguf +model_alias = deepseek +container_name = llama_cpp_deepseek +host_port = 8003 +gpu_device = 1 +ctx_size = 131072 + +[prompt.concise] +system_prompt = Du antwortest kurz, präzise und technisch. + +[prompt.coding] +system_prompt = Du bist ein erfahrener Linux-, Python- und LLM-Infrastruktur-Engineer. diff --git a/llama.cpp.config.example b/llama.cpp.config.example new file mode 100644 index 0000000..aa9f8c8 --- /dev/null +++ b/llama.cpp.config.example @@ -0,0 +1,79 @@ +# llama.cpp.config.example +# +# Copy this file to llama.cpp.config and adjust it for your host before +# running llamacppctl. See docs/SECURITY_AND_OPERATIONS.md for details on +# every field and on the security model for --system-url/--prompt-url. +# +# Section types: +# [default] global defaults, merged over the built-in fallbacks +# [model.] a model profile, selected via --profile +# [prompt.] a system-prompt profile, selected via --system-prompt-profile +# +# container_name is MANDATORY and must be unique per profile you intend to +# run concurrently: it is the single identity anchor for Docker naming, +# locking (--change), and --stop/--check targeting. + +[default] +image = ghcr.io/ggml-org/llama.cpp:server-cuda +# hf_home unterstützt Environment-Variablen und ~, z. B. hf_home = ${HF_HOME} +hf_home = /srv/models +model_path = qwen3/default.gguf +container_name = llama_cpp_server +host_port = 8001 +container_port = 8000 +model_alias = default_llm +gpu_device = 0 +restart_policy = unless-stopped +ctx_size = 262144 +n_predict = 16384 +temp = 0.65 +top_p = 0.80 +top_k = 20 +min_p = 0.01 +repeat_penalty = 1.05 +main_gpu = 0 +ngl = 999 +fa = true +kv_unified = true +jinja = true +reasoning = on +no_context_shift = true +cache_type_k = q4_0 +cache_type_v = q4_0 +batch_size = 1024 +ubatch_size = 512 +parallel = 1 +cont_batching = true +health_endpoint = /health +models_endpoint = /v1/models +chat_endpoint = /v1/chat/completions +timeout = 300 +poll_interval = 2 +# Chat-Antwortbudget (wirkt auf --chat / --start-Antwort, nicht auf den Container). +# Reasoning-Modelle brauchen viel Budget; für lange Texte hochsetzen. +max_tokens = 2048 +# chat_temperature leer lassen -> die Server-Temperatur (temp) gilt. +chat_temperature = + +# Example second model profile, pinned to the second GPU (e.g. RTX 3090 #2) +# with its own port and container name so it can run alongside [default]. +[model.qwen35] +model_path = qwen3/Qwen3-35B-A3B-Q4_K_M.gguf +model_alias = qwen35 +container_name = llama_cpp_qwen35 +host_port = 8002 +gpu_device = 1 + +[model.deepseek] +model_path = deepseek/deepseek-r1-q4.gguf +model_alias = deepseek +container_name = llama_cpp_deepseek +host_port = 8003 +gpu_device = 1 +ctx_size = 131072 + +[prompt.concise] +system_prompt = Du antwortest kurz, präzise und technisch. + +[prompt.coding] +system_prompt = Du bist ein erfahrener Linux-, Python- und LLM-Infrastruktur-Engineer. diff --git a/man/llamacppctl.1 b/man/llamacppctl.1 new file mode 100644 index 0000000..ff78c7c --- /dev/null +++ b/man/llamacppctl.1 @@ -0,0 +1,292 @@ +.TH LLAMACPPCTL 1 "2026-07-05" "llamacppctl 0.1.0" "User Commands" +.SH NAME +llamacppctl \- start, check, stop, switch, or prompt a llama.cpp server running as a Docker container +.SH SYNOPSIS +.B llamacppctl +\fB\-\-start\fR|\fB\-\-check\fR|\fB\-\-stop\fR|\fB\-\-change\fR|\fB\-\-chat\fR +[\fIOPTIONS\fR] +.SH DESCRIPTION +.B llamacppctl +manages a +.B llama.cpp +OpenAI-compatible inference server running inside a Docker container. It +replaces a collection of shell scripts with a single Python CLI that shares +one configuration model, one locking mechanism, and one security-hardened +prompt input layer for system and user prompts. +.SH ACTIONS +Exactly one action is required. +.TP +.B \-\-start +Start the llama.cpp server container. Removes any pre-existing container of +the same name first. Waits for readiness via real chat completions, not just +an open port. +.TP +.B \-\-check +Report Docker container status, HTTP health/model endpoint reachability, and +(if a user prompt is supplied) whether a live chat completion succeeds. +.TP +.B \-\-stop +Stop and remove the container. No-op if it does not exist. +.TP +.B \-\-change +Stop, reconfigure, and restart the server under an exclusive file lock keyed +on \fBcontainer_name\fR, preventing concurrent modification of the same +container. +.TP +.B \-\-chat +Send a single chat completion request to an already-running server and print +the response. +.SH CONFIGURATION MODEL +Configuration is a standard INI file (default path: \fIllama.cpp.config\fR, +override with \fB\-\-config\fR). Resolution order, lowest to highest +priority: +.IP 1. 4 +Built-in defaults +.IP 2. 4 +\fB[default]\fR section +.IP 3. 4 +\fB[model.]\fR section, selected via \fB\-\-profile\fR +.IP 4. 4 +CLI overrides (\fB\-\-image\fR, \fB\-\-host\-port\fR, ...) +.PP +\fB[prompt.]\fR sections, selected via \fB\-\-system\-prompt\-profile\fR, +provide only a \fIfallback\fR system prompt; an explicit +\fB\-s\fR/\fB\-\-system\-file\fR/\fB\-\-system\-url\fR always wins. +.PP +\fBcontainer_name\fR is mandatory and must be non-empty after resolution. It +is the single identity anchor for Docker naming, \fB\-\-stop\fR/\fB\-\-check\fR +targeting, and the derived lock file path +(\fI/tmp/llamacppctl..lock\fR). Profiles intended to run +concurrently must each use a distinct \fBcontainer_name\fR. +.SH OPTIONS +.SS General +.TP +.B \-\-config \fIPATH\fR +Path to the INI configuration file. Default: \fIllama.cpp.config\fR. +.TP +.B \-\-profile \fINAME\fR +Select a \fB[model.]\fR section. +.TP +.B \-\-system\-prompt\-profile \fINAME\fR +Select a \fB[prompt.]\fR section for a fallback system prompt. +.SS Prompt sources (system) +Mutually exclusive. +.TP +.B \-s\fR, \fB\-\-system \fITEXT\fR +Literal system prompt text. +.TP +.B \-\-system\-file \fIPATH\fR +Read the system prompt from a local file. +.TP +.B \-\-system\-url \fIURL\fR +Read the system prompt from a remote HTTPS URL. +.SS Prompt sources (user) +Mutually exclusive. +.TP +.B \-p\fR, \fB\-\-prompt \fITEXT\fR +Literal user prompt text. +.TP +.B \-\-prompt\-file \fIPATH\fR +Read the user prompt from a local file. +.TP +.B \-\-prompt\-url \fIURL\fR +Read the user prompt from a remote HTTPS URL. +.SS Docker / server overrides +.TP +.B \-\-image \fIIMAGE\fR +Docker image, e.g. \fIghcr.io/ggml-org/llama.cpp:server-cuda\fR. +.TP +.B \-\-hf\-home \fIPATH\fR +Host path mounted read-only into the container as model storage. Environment +variables (e.g. \fB${HF_HOME}\fR) and \fB~\fR are expanded, so the config may +contain \fBhf_home = ${HF_HOME}\fR. +.TP +.B \-\-model\-path \fIPATH\fR +Model path, relative to \fB\-\-hf\-home\fR unless absolute. +.TP +.B \-\-container\-name \fINAME\fR +Docker container name. Must be unique per concurrently-running profile. +.TP +.B \-\-host\-port \fIPORT\fR\fR, \fB\-\-container\-port \fIPORT\fR +Host / container-internal TCP port (1-65535). +.TP +.B \-\-model\-alias \fINAME\fR +Model alias exposed via the OpenAI-compatible API. +.TP +.B \-\-gpu \fIID\fR +GPU device id passed to \fB\-\-gpus device=\fR. +.TP +.B \-\-restart\-policy \fIPOLICY\fR +Docker \fB\-\-restart\fR policy. +.SS llama.cpp runtime parameters +.B \-\-ctx\-size\fR, \fB\-\-n\-predict\fR, \fB\-\-temp\fR, \fB\-\-top\-p\fR, +\fB\-\-top\-k\fR, \fB\-\-min\-p\fR, \fB\-\-repeat\-penalty\fR, +\fB\-\-main\-gpu\fR, \fB\-\-ngl\fR, \fB\-\-batch\-size\fR, +\fB\-\-ubatch\-size\fR, \fB\-\-parallel\fR, \fB\-\-cache\-type\-k\fR, +\fB\-\-cache\-type\-v\fR, \fB\-\-reasoning {on,off}\fR. +.PP +Boolean pairs (unset by default, meaning "use config/default value"): +\fB\-\-jinja\fR/\fB\-\-no\-jinja\fR, \fB\-\-fa\fR/\fB\-\-no\-fa\fR, +\fB\-\-kv\-unified\fR/\fB\-\-no\-kv\-unified\fR, +\fB\-\-cont\-batching\fR/\fB\-\-no\-cont\-batching\fR, +\fB\-\-no\-context\-shift\fR/\fB\-\-context\-shift\fR. +.SS Chat request parameters +Applied to the reply produced by \fB\-\-chat\fR and the post-start reply of +\fB\-\-start\fR (they do not affect the container itself). Config keys: +\fBmax_tokens\fR, \fBchat_temperature\fR. +.TP +.B \-\-max\-tokens \fIN\fR +Maximum tokens for the chat reply. Default 2048; raise it for reasoning models +or long-form output (they spend many tokens "thinking" before the visible +answer, so a small budget yields empty content). Must be > 0. +.TP +.B \-\-chat\-temp \fIFLOAT\fR +Temperature for the chat request. If unset, the request omits temperature and +the server's configured \fB\-\-temp\fR applies (single source of truth). +.SS Security / behavior flags (prompt input) +.TP +.B \-\-allow\-insecure\-http +Permit \fBhttp://\fR URLs for \fB\-\-system\-url\fR/\fB\-\-prompt\-url\fR (default: HTTPS only). +.TP +.B \-\-allow\-private\-url +Disable private/loopback/link-local/reserved/metadata IP blocking for URL +sources. Cannot be combined with \fB\-\-url\-allow\-host\fR. +.TP +.B \-\-allow\-ip\-host +Permit IP-literal hostnames in URLs (default: rejected). +.TP +.B \-\-follow\-redirects +Follow HTTP redirects for URL sources (default: rejected). Each hop is +re-validated against the same SSRF rules. Bounded by an internal max-redirect +count. +.TP +.B \-\-allow\-html\-input +Accept \fBtext/html\fR responses in addition to the default text-like +content-type allowlist. +.TP +.B \-\-allow\-symlinks\fR / \fB\-\-no\-allow\-symlinks +Allow or reject symlinked files for \fB\-\-system\-file\fR/\fB\-\-prompt\-file\fR. +Default: rejected (opt in with \fB\-\-allow\-symlinks\fR). +.TP +.B \-\-max\-input\-bytes \fIN\fR +Maximum size in bytes for any resolved prompt input (file or URL). Default: +1048576 (1 MiB). Must be > 0. +.TP +.B \-\-url\-allow\-host \fIHOST\fR +Restrict URL sources to an explicit host allowlist. Repeatable. +.TP +.B \-\-connect\-timeout \fISECONDS\fR\fR, \fB\-\-read\-timeout \fISECONDS\fR +Network timeouts for URL sources. Must be > 0. Defaults: 3.0 / 10.0. +.SS Behavior / diagnostics +.TP +.B \-\-timeout \fISECONDS\fR +Readiness timeout for \fB\-\-start\fR/\fB\-\-change\fR. Default from config +(300s). +.TP +.B \-\-poll\-interval \fISECONDS\fR +Readiness poll interval. Default from config (2s). +.TP +.B \-\-lock\-file \fIPATH\fR +Override the lock file path used by \fB\-\-start\fR/\fB\-\-change\fR. Default: +derived from \fBcontainer_name\fR. +.TP +.B \-\-logs +Print container log tail on readiness failure. +.TP +.B \-\-log\-lines \fIN\fR +Number of log lines to print. Default: 100. +.TP +.B \-\-dry\-run +Print the fully constructed \fBdocker run\fR command and exit without +executing it. Usable with \fB\-\-start\fR/\fB\-\-change\fR. +.TP +.B \-\-force +Force stop/change despite inconsistencies. +.TP +.B \-\-print\-effective\-config +Print the fully merged configuration as JSON and exit (unless combined with +an action that also runs). +.SH SECURITY NOTES +All Docker interaction goes through \fBsubprocess.run()\fR with argument +lists, never shell strings — there is no shell-injection surface via +configuration values or CLI arguments. +.PP +The remote URL prompt source (\fB\-\-system\-url\fR/\fB\-\-prompt\-url\fR) is +SSRF-hardened by default: HTTPS-only, no IP-literal hosts, no embedded +credentials, DNS resolution and IP classification for every candidate +address (blocking private/loopback/link-local/multicast/reserved/ +unspecified ranges and the cloud metadata address +\fI169.254.169.254\fR), no redirects unless explicitly enabled (with +per-hop re-validation), a content-type allowlist, and a hard size limit +enforced both via \fBContent-Length\fR and while streaming the response body. +See \fIdocs/SECURITY_AND_OPERATIONS.md\fR in the source distribution for the +full threat model. +.PP +\fBcontainer_name\fR is mandatory and must be unique per concurrently-running +profile; reusing a name across profiles that run at the same time will cause +one to replace the other's container on the next \fB\-\-start\fR. +.SH FILES +.TP +.I llama.cpp.config +Default configuration file (INI format). See \fIllama.cpp.config.example\fR +in the source distribution. +.TP +.I /tmp/llamacppctl..lock +Default lock file used by \fB\-\-change\fR, derived from \fBcontainer_name\fR +unless \fB\-\-lock\-file\fR is set. +.SH EXIT STATUS +.TP +.B 0 +Success. +.TP +.B 1 +General error (Docker unavailable, unexpected exception, prompt input error, configuration error). +.TP +.B 2 +CLI argument / semantic validation error. +.TP +.B 3 +Model path not found on disk. +.TP +.B 4 +Docker command failed. +.TP +.B 5 +HTTP request failed or server did not become ready in time. +.TP +.B 6 +Lock busy (a concurrent \fB\-\-change\fR is already running for the same \fBcontainer_name\fR). +.SH EXAMPLES +.PP +Start a profile and show the effective configuration first: +.RS +.nf +llamacppctl \-\-print\-effective\-config \-\-config llama.cpp.config \-\-profile qwen35 \-\-start +llamacppctl \-\-start \-\-config llama.cpp.config \-\-profile qwen35 +.fi +.RE +.PP +Preview the docker run command without executing it: +.RS +.nf +llamacppctl \-\-start \-\-config llama.cpp.config \-\-profile qwen35 \-\-dry\-run +.fi +.RE +.PP +Check status including a live chat completion: +.RS +.nf +llamacppctl \-\-check \-\-config llama.cpp.config \-\-profile qwen35 \-p "ping" +.fi +.RE +.PP +Switch models under lock protection: +.RS +.nf +llamacppctl \-\-change \-\-config llama.cpp.config \-\-profile deepseek +.fi +.RE +.SH AUTHOR +Dieter Schlueter +.SH SEE ALSO +.BR docker (1) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4a3e8d0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "llamacppctl" +version = "0.1.0" +description = "CLI to start, check, stop, and switch llama.cpp Docker server instances with a security-hardened prompt input layer." +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "Dieter Schlueter" }] +dependencies = [ + "requests>=2.31,<3", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4", +] + +[project.scripts] +llamacppctl = "llamacppctl.main:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +llamacppctl = ["py.typed"] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..167e7b9 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +# Development/test dependencies (mirrors [project.optional-dependencies].dev +# in pyproject.toml). +-r requirements.txt +pytest>=7.4 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b3cd6be --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +# Runtime dependencies (mirrors [project.dependencies] in pyproject.toml). +# Provided for users who want to verify/install dependencies without pip's +# PEP 517 build isolation, e.g. `pip install -r requirements.txt`. +requests>=2.31,<3 diff --git a/src/llamacppctl/__init__.py b/src/llamacppctl/__init__.py new file mode 100644 index 0000000..2ccf30b --- /dev/null +++ b/src/llamacppctl/__init__.py @@ -0,0 +1,5 @@ +"""llamacppctl - a general-purpose CLI to start, check, stop and switch llama.cpp +server containers, with a strict, explicit prompt-input layer (literal / file / URL). +""" + +__version__ = "0.1.0" diff --git a/src/llamacppctl/actions.py b/src/llamacppctl/actions.py new file mode 100644 index 0000000..7072718 --- /dev/null +++ b/src/llamacppctl/actions.py @@ -0,0 +1,193 @@ +"""Top-level action implementations: do_start, do_check, do_stop, do_change, do_chat. + +Exit code convention: + 0 success + 1 general error (docker unavailable, unexpected exception) + 2 CLI / validation error (handled by argparse.error before reaching here) + 3 model path missing + 4 docker error + 5 HTTP / readiness not ready + 6 lock busy +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import asdict +from pathlib import Path + +from .docker_ops import ( + DockerError, + build_run_command, + container_logs, + docker_available, + format_command_for_display, + inspect_container, + remove_container, + start_llama_container, + stop_container, +) +from .http_ops import HttpError, base_url, chat_completion_text, wait_until_ready +from .lock_ops import FileLock, LockError +from .schema import CheckResult, PromptConfig, ServerConfig + +EXIT_OK = 0 +EXIT_GENERAL = 1 +EXIT_MODEL_PATH_MISSING = 3 +EXIT_DOCKER_ERROR = 4 +EXIT_HTTP_NOT_READY = 5 +EXIT_LOCK_BUSY = 6 + + +def validate_model_path(cfg: ServerConfig) -> None: + model_path = Path(cfg.model_path) + if model_path.is_absolute(): + target = model_path + else: + target = cfg.hf_home / cfg.model_path + if not target.exists(): + raise FileNotFoundError(f"model file not found: {target}") + + +def print_effective_config(cfg: ServerConfig, prompt_cfg: PromptConfig) -> None: + payload = { + "server": {k: (str(v) if isinstance(v, Path) else v) for k, v in asdict(cfg).items()}, + "prompt": asdict(prompt_cfg), + } + print(json.dumps(payload, indent=2, ensure_ascii=False)) + + +def do_check(cfg: ServerConfig, prompt_cfg: PromptConfig, args) -> CheckResult: + info = inspect_container(cfg.container_name) + exists = info.status != "missing" + running = info.status == "running" + + from .http_ops import check_health, check_models + + health_ok = check_health(cfg) if running else False + models_ok = check_models(cfg) if running else False + chat_ok = False + + if running and prompt_cfg.user_prompt: + try: + chat_completion_text(cfg, prompt_cfg) + chat_ok = True + except Exception: + chat_ok = False + + return CheckResult( + container_exists=exists, + container_running=running, + http_ok=health_ok or models_ok, + models_ok=models_ok, + chat_ok=chat_ok, + base_url=base_url(cfg), + detail=f"status={info.status} health={info.health}", + ) + + +def print_check_result(cfg: ServerConfig, result: CheckResult) -> None: + print(f"Container: {cfg.container_name}") + print(f"Docker: {'RUNNING' if result.container_running else ('EXISTS (stopped)' if result.container_exists else 'NOT FOUND')}") + print(f"HTTP: {'OK' if result.http_ok else 'UNREACHABLE'}") + print(f"Base URL: {result.base_url}") + print(f"Model: {cfg.model_alias}") + if result.chat_ok: + print("Chat completions: OK") + print(f"Detail: {result.detail}") + + +def do_stop(cfg: ServerConfig, args) -> int: + try: + removed = stop_container(cfg.container_name, remove=True) + except DockerError as exc: + print(f"docker error: {exc}", file=sys.stderr) + return EXIT_DOCKER_ERROR + if removed: + print(f"Stopped and removed container: {cfg.container_name}") + else: + print(f"Container not found (nothing to stop): {cfg.container_name}") + return EXIT_OK + + +def do_start(cfg: ServerConfig, prompt_cfg: PromptConfig, args) -> int: + """Start under the container's file lock so concurrent --start/--change + invocations cannot race on the same container_name.""" + if getattr(args, "dry_run", False): + print(format_command_for_display(build_run_command(cfg))) + return EXIT_OK + try: + with FileLock(cfg.lock_file): + return _start_locked(cfg, prompt_cfg, args) + except LockError as exc: + print(str(exc), file=sys.stderr) + return EXIT_LOCK_BUSY + + +def _start_locked(cfg: ServerConfig, prompt_cfg: PromptConfig, args) -> int: + """Core start logic. The caller must already hold the container lock.""" + try: + validate_model_path(cfg) + except FileNotFoundError as exc: + print(str(exc), file=sys.stderr) + return EXIT_MODEL_PATH_MISSING + + try: + container_id = start_llama_container(cfg) + except DockerError as exc: + print(f"docker error: {exc}", file=sys.stderr) + return EXIT_DOCKER_ERROR + + print(f"Started container {cfg.container_name} ({container_id[:12]})") + print(f"Waiting for readiness (timeout={cfg.timeout}s)...") + + probe = PromptConfig(system_prompt=None, user_prompt="ping", max_tokens=1, temperature=0.0, stream=False) + if not wait_until_ready(cfg, probe): + print(f"Server did not become ready within {cfg.timeout}s", file=sys.stderr) + if getattr(args, "logs", False): + print("---- container logs (tail) ----", file=sys.stderr) + print(container_logs(cfg.container_name, getattr(args, "log_lines", 100)), file=sys.stderr) + return EXIT_HTTP_NOT_READY + + print("Model ready.") + + if prompt_cfg.user_prompt: + try: + content = chat_completion_text(cfg, prompt_cfg) + print("---- response ----") + print(content) + except HttpError as exc: + print(f"chat request failed: {exc}", file=sys.stderr) + + return EXIT_OK + + +def do_change(cfg: ServerConfig, prompt_cfg: PromptConfig, args) -> int: + # Dry-run is a pure preview: never take the lock or remove the container. + if getattr(args, "dry_run", False): + print(format_command_for_display(build_run_command(cfg))) + return EXIT_OK + try: + with FileLock(cfg.lock_file): + remove_container(cfg.container_name) + return _start_locked(cfg, prompt_cfg, args) + except LockError as exc: + print(str(exc), file=sys.stderr) + return EXIT_LOCK_BUSY + except DockerError as exc: + print(f"docker error: {exc}", file=sys.stderr) + return EXIT_DOCKER_ERROR + + +def do_chat(cfg: ServerConfig, prompt_cfg: PromptConfig, args) -> int: + if not prompt_cfg.user_prompt: + print("--chat requires a user prompt", file=sys.stderr) + return EXIT_GENERAL + try: + content = chat_completion_text(cfg, prompt_cfg) + except HttpError as exc: + print(f"chat request failed: {exc}", file=sys.stderr) + return EXIT_HTTP_NOT_READY + print(content) + return EXIT_OK diff --git a/src/llamacppctl/cli.py b/src/llamacppctl/cli.py new file mode 100644 index 0000000..6755c94 --- /dev/null +++ b/src/llamacppctl/cli.py @@ -0,0 +1,165 @@ +"""Argument parsing and CLI-level semantic validation for llamacppctl.""" + +from __future__ import annotations + +import argparse + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="llamacppctl", + description=( + "Start, check, stop, switch or prompt a llama.cpp server " + "running as a Docker container." + ), + ) + + action = p.add_mutually_exclusive_group(required=True) + action.add_argument("--start", action="store_true", help="Start the llama.cpp server") + action.add_argument("--check", action="store_true", help="Check container and HTTP status") + action.add_argument("--stop", action="store_true", help="Stop/remove the container") + action.add_argument("--change", action="store_true", help="Stop, reconfigure, and restart") + action.add_argument("--chat", action="store_true", help="Send a prompt to a running server") + + p.add_argument("--config", default="llama.cpp.config", help="Path to llama.cpp.config") + p.add_argument("--profile", help="Model profile name: [model.] in config") + p.add_argument( + "--system-prompt-profile", + help="Prompt profile name: [prompt.] in config (default system prompt)", + ) + + # --- Prompt sources: exactly one source per field, enforced by argparse + # mutually-exclusive groups plus a semantic validation pass below. --- + system_src = p.add_argument_group("system prompt source") + system_x = system_src.add_mutually_exclusive_group() + system_x.add_argument("-s", "--system", help="Literal system prompt text") + system_x.add_argument("--system-file", help="Read system prompt from a local file") + system_x.add_argument("--system-url", help="Read system prompt from a remote HTTPS URL") + + prompt_src = p.add_argument_group("user prompt source") + prompt_x = prompt_src.add_mutually_exclusive_group() + prompt_x.add_argument("-p", "--prompt", help="Literal user prompt text") + prompt_x.add_argument("--prompt-file", help="Read user prompt from a local file") + prompt_x.add_argument("--prompt-url", help="Read user prompt from a remote HTTPS URL") + + # --- Docker / server overrides --- + p.add_argument("--image", help="Docker image (e.g. ghcr.io/ggml-org/llama.cpp:server-cuda)") + p.add_argument("--hf-home", help="Host path mounted read-only as model storage") + p.add_argument("--model-path", help="Model path, relative to --hf-home unless absolute") + p.add_argument("--container-name", help="Docker container name (must be unique per config)") + p.add_argument("--host-port", type=int, help="Host port to publish") + p.add_argument("--container-port", type=int, help="Container-internal port") + p.add_argument("--model-alias", help="Model alias exposed via the OpenAI-compatible API") + p.add_argument("--gpu", dest="gpu_device", help="GPU device id passed to --gpus device=") + p.add_argument("--restart-policy", help="Docker --restart policy") + + # --- llama.cpp runtime parameters --- + p.add_argument("--ctx-size", type=int) + p.add_argument("--n-predict", type=int) + p.add_argument("--temp", type=float) + p.add_argument("--top-p", type=float) + p.add_argument("--top-k", type=int) + p.add_argument("--min-p", type=float) + p.add_argument("--repeat-penalty", type=float) + p.add_argument("--main-gpu", type=int) + p.add_argument("--ngl", type=int) + p.add_argument("--batch-size", type=int) + p.add_argument("--ubatch-size", type=int) + p.add_argument("--parallel", type=int) + p.add_argument("--cache-type-k") + p.add_argument("--cache-type-v") + p.add_argument("--reasoning", choices=["on", "off"]) + + p.add_argument("--jinja", dest="jinja", action="store_true", default=None) + p.add_argument("--no-jinja", dest="jinja", action="store_false") + p.add_argument("--fa", dest="fa", action="store_true", default=None) + p.add_argument("--no-fa", dest="fa", action="store_false") + p.add_argument("--kv-unified", dest="kv_unified", action="store_true", default=None) + p.add_argument("--no-kv-unified", dest="kv_unified", action="store_false") + p.add_argument("--cont-batching", dest="cont_batching", action="store_true", default=None) + p.add_argument("--no-cont-batching", dest="cont_batching", action="store_false") + p.add_argument("--no-context-shift", dest="no_context_shift", action="store_true", default=None) + p.add_argument("--context-shift", dest="no_context_shift", action="store_false") + + # --- Prompt-input security / behavior options --- + p.add_argument("--allow-insecure-http", action="store_true") + p.add_argument("--allow-private-url", action="store_true") + p.add_argument("--allow-ip-host", action="store_true") + p.add_argument("--follow-redirects", action="store_true") + p.add_argument("--allow-html-input", action="store_true") + p.add_argument("--allow-symlinks", dest="allow_symlinks", action="store_true", default=False) + p.add_argument("--no-allow-symlinks", dest="allow_symlinks", action="store_false") + p.add_argument("--max-input-bytes", type=int, default=1_048_576) + p.add_argument("--url-allow-host", action="append", default=[]) + p.add_argument("--connect-timeout", type=float, default=3.0) + p.add_argument("--read-timeout", type=float, default=10.0) + + # --- Chat request parameters (for --chat and the --start reply) --- + p.add_argument( + "--max-tokens", + type=int, + help="Max tokens for the --chat / --start reply (default 2048; raise for reasoning models)", + ) + p.add_argument( + "--chat-temp", + type=float, + help="Temperature for the chat request (default: use the server's configured --temp)", + ) + + # --- Behavior / diagnostics --- + p.add_argument("--timeout", type=int, help="Readiness timeout in seconds") + p.add_argument("--poll-interval", type=float, help="Readiness poll interval in seconds") + p.add_argument("--lock-file", help="Override lock file path (default derived from container_name)") + p.add_argument("--logs", action="store_true", help="Print container logs on readiness failure") + p.add_argument("--log-lines", type=int, default=100) + p.add_argument("--dry-run", action="store_true", help="Show the docker run command, do not execute it") + p.add_argument("--force", action="store_true", help="Force stop/change despite inconsistencies") + p.add_argument( + "--print-effective-config", + action="store_true", + help="Print the fully merged configuration as JSON and exit (unless combined with an action)", + ) + + return p + + +def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: + """Enforce semantic rules that argparse's mutually-exclusive groups alone + cannot express (cross-field and cross-action relationships).""" + + has_system_source = any( + [args.system is not None, args.system_file is not None, args.system_url is not None] + ) + has_prompt_source = any( + [args.prompt is not None, args.prompt_file is not None, args.prompt_url is not None] + ) + + if args.chat and not has_prompt_source: + parser.error("--chat requires one of --prompt, --prompt-file, or --prompt-url") + + if args.stop and (has_system_source or has_prompt_source): + parser.error("--stop does not accept prompt input sources") + + if args.check and has_system_source and not has_prompt_source: + parser.error("a system prompt source without a prompt source is not useful for --check") + + if args.max_input_bytes <= 0: + parser.error("--max-input-bytes must be > 0") + + if args.max_tokens is not None and args.max_tokens <= 0: + parser.error("--max-tokens must be > 0") + + if args.connect_timeout <= 0 or args.read_timeout <= 0: + parser.error("timeouts must be > 0") + + if args.allow_private_url and args.url_allow_host: + parser.error("--allow-private-url cannot be combined with --url-allow-host") + + if args.host_port is not None and not (1 <= args.host_port <= 65535): + parser.error("--host-port must be between 1 and 65535") + + if args.container_port is not None and not (1 <= args.container_port <= 65535): + parser.error("--container-port must be between 1 and 65535") + + if args.container_name is not None and not args.container_name.strip(): + parser.error("--container-name must not be empty") diff --git a/src/llamacppctl/config.py b/src/llamacppctl/config.py new file mode 100644 index 0000000..bc00a5a --- /dev/null +++ b/src/llamacppctl/config.py @@ -0,0 +1,249 @@ +"""llama.cpp.config loading and merging. + +Resolution order (lowest to highest priority): + 1. Built-in defaults (mirrors the user's original shell-script defaults) + 2. [default] section in llama.cpp.config + 3. [model.] section (selected via --profile) + 4. CLI overrides (--image, --host-port, ...) + +[prompt.] sections are resolved separately and only ever provide a +*default* system prompt, which explicit --system/--system-file/--system-url +CLI sources always override. + +container_name is mandatory and must be non-empty after resolution: it is +the single identity anchor used for locking, start/stop/change and Docker +container naming. Config authors that define multiple [model.*] profiles +intended to run concurrently MUST give each one a distinct container_name. +""" + +from __future__ import annotations + +import configparser +import os +from pathlib import Path +from typing import Optional + +from .schema import PromptConfig, ServerConfig + +TRUE_STRINGS = {"1", "true", "yes", "on"} +FALSE_STRINGS = {"0", "false", "no", "off"} + + +class ConfigError(ValueError): + """Raised for missing profiles, invalid values, or invalid container_name.""" + + +def builtin_defaults() -> dict: + """Mirrors the original start-llm-server.sh / status-llm-server.sh defaults.""" + return { + "image": "ghcr.io/ggml-org/llama.cpp:server-cuda", + "hf_home": "/models", + "model_path": "qwen3/default.gguf", + "container_name": "va_llm", + "host_port": "8001", + "container_port": "8000", + "model_alias": "va_llm", + "gpu_device": "1", + "restart_policy": "unless-stopped", + "ctx_size": "262144", + "n_predict": "16384", + "temp": "0.65", + "top_p": "0.80", + "top_k": "20", + "min_p": "0.01", + "repeat_penalty": "1.05", + "main_gpu": "0", + "ngl": "999", + "cache_type_k": "q4_0", + "cache_type_v": "q4_0", + "batch_size": "1024", + "ubatch_size": "512", + "parallel": "1", + "reasoning": "on", + "jinja": "true", + "fa": "true", + "kv_unified": "true", + "cont_batching": "true", + "no_context_shift": "true", + "host": "0.0.0.0", + "health_endpoint": "/health", + "models_endpoint": "/v1/models", + "chat_endpoint": "/v1/chat/completions", + "timeout": "300", + "poll_interval": "2", + "lock_file": "", # derived from container_name if empty + # Chat-request defaults (PromptConfig, not container flags): + "max_tokens": "2048", # reply budget; raise for reasoning/long output + "chat_temperature": "", # empty => defer to the server's --temp + } + + +def load_ini(path: Path) -> configparser.ConfigParser: + cp = configparser.ConfigParser() + if path.exists(): + cp.read(path, encoding="utf-8") + return cp + + +def _to_bool(value, field_name: str) -> bool: + if isinstance(value, bool): + return value + s = str(value).strip().lower() + if s in TRUE_STRINGS: + return True + if s in FALSE_STRINGS: + return False + raise ConfigError(f"invalid boolean for {field_name!r}: {value!r}") + + +def _apply_cli_overrides(merged: dict, args) -> None: + overrides = { + "image": getattr(args, "image", None), + "hf_home": getattr(args, "hf_home", None), + "model_path": getattr(args, "model_path", None), + "container_name": getattr(args, "container_name", None), + "host_port": getattr(args, "host_port", None), + "container_port": getattr(args, "container_port", None), + "model_alias": getattr(args, "model_alias", None), + "gpu_device": getattr(args, "gpu_device", None), + "restart_policy": getattr(args, "restart_policy", None), + "ctx_size": getattr(args, "ctx_size", None), + "n_predict": getattr(args, "n_predict", None), + "temp": getattr(args, "temp", None), + "top_p": getattr(args, "top_p", None), + "top_k": getattr(args, "top_k", None), + "min_p": getattr(args, "min_p", None), + "repeat_penalty": getattr(args, "repeat_penalty", None), + "main_gpu": getattr(args, "main_gpu", None), + "ngl": getattr(args, "ngl", None), + "batch_size": getattr(args, "batch_size", None), + "ubatch_size": getattr(args, "ubatch_size", None), + "parallel": getattr(args, "parallel", None), + "cache_type_k": getattr(args, "cache_type_k", None), + "cache_type_v": getattr(args, "cache_type_v", None), + "reasoning": getattr(args, "reasoning", None), + "jinja": getattr(args, "jinja", None), + "fa": getattr(args, "fa", None), + "kv_unified": getattr(args, "kv_unified", None), + "cont_batching": getattr(args, "cont_batching", None), + "no_context_shift": getattr(args, "no_context_shift", None), + "timeout": getattr(args, "timeout", None), + "poll_interval": getattr(args, "poll_interval", None), + "lock_file": getattr(args, "lock_file", None), + } + for key, value in overrides.items(): + if value is not None: + merged[key] = value + + +def build_server_config(merged: dict) -> ServerConfig: + container_name = str(merged.get("container_name", "")).strip() + if not container_name: + raise ConfigError( + "container_name is mandatory and must not be empty " + "(set it in [default], in a [model.] section, or via --container-name)" + ) + + # Support environment variables (e.g. hf_home = ${HF_HOME}) and ~ in paths. + hf_home = Path(os.path.expanduser(os.path.expandvars(str(merged["hf_home"])))) + + lock_file_value = str(merged.get("lock_file") or "").strip() + if lock_file_value: + lock_file = Path(lock_file_value) + else: + lock_file = Path(f"/tmp/llamacppctl.{container_name}.lock") + + return ServerConfig( + image=str(merged["image"]), + hf_home=hf_home, + model_path=str(merged["model_path"]), + container_name=container_name, + host_port=int(merged["host_port"]), + container_port=int(merged["container_port"]), + model_alias=str(merged["model_alias"]), + gpu_device=str(merged["gpu_device"]), + restart_policy=str(merged["restart_policy"]), + ctx_size=int(merged["ctx_size"]), + n_predict=int(merged["n_predict"]), + temp=float(merged["temp"]), + top_p=float(merged["top_p"]), + top_k=int(merged["top_k"]), + min_p=float(merged["min_p"]), + repeat_penalty=float(merged["repeat_penalty"]), + main_gpu=int(merged["main_gpu"]), + ngl=int(merged["ngl"]), + batch_size=int(merged["batch_size"]), + ubatch_size=int(merged["ubatch_size"]), + parallel=int(merged["parallel"]), + cache_type_k=str(merged["cache_type_k"]), + cache_type_v=str(merged["cache_type_v"]), + reasoning="on" if str(merged["reasoning"]).lower() in ("on", "true", "1", "yes") else "off", + jinja=_to_bool(merged["jinja"], "jinja"), + fa=_to_bool(merged["fa"], "fa"), + kv_unified=_to_bool(merged["kv_unified"], "kv_unified"), + cont_batching=_to_bool(merged["cont_batching"], "cont_batching"), + no_context_shift=_to_bool(merged["no_context_shift"], "no_context_shift"), + host=str(merged["host"]), + health_endpoint=str(merged["health_endpoint"]), + models_endpoint=str(merged["models_endpoint"]), + chat_endpoint=str(merged["chat_endpoint"]), + timeout=int(merged["timeout"]), + poll_interval=float(merged["poll_interval"]), + lock_file=lock_file, + ) + + +def resolve_effective_config(args) -> tuple: + """Returns (ServerConfig, PromptConfig) fully resolved from + builtin defaults -> [default] -> [model.] -> CLI overrides, + with [prompt.] providing only a fallback system prompt.""" + + defaults = builtin_defaults() + cp = load_ini(Path(args.config)) + + merged = dict(defaults) + + if cp.has_section("default"): + merged.update(dict(cp["default"])) + + profile = getattr(args, "profile", None) + if profile: + sec = f"model.{profile}" + if not cp.has_section(sec): + raise ConfigError(f"model profile not found: {profile}") + merged.update(dict(cp[sec])) + + prompt_defaults = {} + prompt_profile = getattr(args, "system_prompt_profile", None) + if prompt_profile: + sec = f"prompt.{prompt_profile}" + if not cp.has_section(sec): + raise ConfigError(f"prompt profile not found: {prompt_profile}") + prompt_defaults.update(dict(cp[sec])) + + _apply_cli_overrides(merged, args) + + server_cfg = build_server_config(merged) + + system_prompt: Optional[str] = getattr(args, "_resolved_system_prompt", None) + if system_prompt is None: + system_prompt = prompt_defaults.get("system_prompt") + + # Chat-request params: config value first, CLI flag overrides. + max_tokens = int(merged.get("max_tokens") or 2048) + if getattr(args, "max_tokens", None) is not None: + max_tokens = args.max_tokens + + ct_raw = str(merged.get("chat_temperature") or "").strip() + chat_temp: Optional[float] = float(ct_raw) if ct_raw else None + if getattr(args, "chat_temp", None) is not None: + chat_temp = args.chat_temp + + prompt_cfg = PromptConfig( + system_prompt=system_prompt, + user_prompt=getattr(args, "_resolved_user_prompt", None), + max_tokens=max_tokens, + temperature=chat_temp, + ) + + return server_cfg, prompt_cfg diff --git a/src/llamacppctl/docker_ops.py b/src/llamacppctl/docker_ops.py new file mode 100644 index 0000000..ae84077 --- /dev/null +++ b/src/llamacppctl/docker_ops.py @@ -0,0 +1,178 @@ +"""Docker container lifecycle for the llama.cpp server. + +All Docker interaction goes through subprocess.run() with argument lists +(never shell strings), so there is no shell-quoting/injection surface here. +""" + +from __future__ import annotations + +import json +import shlex +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from .schema import ServerConfig + + +class DockerError(RuntimeError): + pass + + +@dataclass +class ContainerInfo: + name: str + status: str # "running", "exited", "missing", "unknown" + health: str # "healthy", "unhealthy", "starting", "none" + + +def _run(cmd: list, check: bool = True) -> subprocess.CompletedProcess: + try: + return subprocess.run(cmd, text=True, capture_output=True, check=check) + except FileNotFoundError as exc: + raise DockerError("docker executable not found on PATH") from exc + except subprocess.CalledProcessError as exc: + raise DockerError( + f"command failed ({' '.join(cmd)}): {exc.stderr or exc.stdout}" + ) from exc + + +def docker_available() -> bool: + try: + cp = subprocess.run(["docker", "version"], text=True, capture_output=True) + return cp.returncode == 0 + except FileNotFoundError: + return False + + +def container_exists(name: str) -> bool: + cp = _run(["docker", "ps", "-a", "--format", "{{.Names}}"], check=True) + return name in cp.stdout.splitlines() + + +def container_running(name: str) -> bool: + cp = _run(["docker", "ps", "--format", "{{.Names}}"], check=True) + return name in cp.stdout.splitlines() + + +def inspect_container(name: str) -> ContainerInfo: + if not container_exists(name): + return ContainerInfo(name=name, status="missing", health="none") + + cp = _run(["docker", "inspect", name], check=True) + try: + data = json.loads(cp.stdout)[0] + except (json.JSONDecodeError, IndexError, KeyError) as exc: + raise DockerError(f"failed to parse docker inspect output for {name}") from exc + + state = data.get("State", {}) + status = state.get("Status", "unknown") + health = state.get("Health", {}).get("Status", "none") + return ContainerInfo(name=name, status=status, health=health) + + +def remove_container(name: str) -> bool: + if not container_exists(name): + return False + _run(["docker", "rm", "-f", name], check=True) + return True + + +def stop_container(name: str, remove: bool = True) -> bool: + """Stop (and by default remove) a container. No-op if it does not exist.""" + if not container_exists(name): + return False + if remove: + _run(["docker", "rm", "-f", name], check=True) + else: + _run(["docker", "stop", name], check=True) + return True + + +def container_logs(name: str, tail: int = 100) -> str: + cp = _run(["docker", "logs", "--tail", str(tail), name], check=False) + return (cp.stdout or "") + (cp.stderr or "") + + +# Backwards-compatible alias matching the original spec naming. +tail_logs = container_logs + + +def _resolve_model_path_in_container(cfg: ServerConfig) -> str: + model_path = Path(cfg.model_path) + if model_path.is_absolute(): + return str(model_path) + return f"/hf_home/{cfg.model_path}" + + +def build_run_command(cfg: ServerConfig) -> list: + """Build the `docker run` argument list for the llama.cpp server. + + Mirrors the original start-llm-server.sh defaults: GPU pinning via + --gpus device=, HF_HOME mounted read-only, OpenAI-compatible + server flags for sampling and context parameters. + """ + model_in_container = _resolve_model_path_in_container(cfg) + + cmd = [ + "docker", "run", "-d", + "--gpus", f"device={cfg.gpu_device}", + "--name", cfg.container_name, + "--restart", cfg.restart_policy, + "-e", "HF_HOME=/hf_home", + "-v", f"{cfg.hf_home}:/hf_home:ro", + "-p", f"{cfg.host_port}:{cfg.container_port}", + cfg.image, + "-m", model_in_container, + "--alias", cfg.model_alias, + "-c", str(cfg.ctx_size), + "-n", str(cfg.n_predict), + "--temp", str(cfg.temp), + "--top-p", str(cfg.top_p), + "--top-k", str(cfg.top_k), + "--min-p", str(cfg.min_p), + "--repeat-penalty", str(cfg.repeat_penalty), + "--main-gpu", str(cfg.main_gpu), + "-ngl", str(cfg.ngl), + "--cache-type-k", cfg.cache_type_k, + "--cache-type-v", cfg.cache_type_v, + "--batch-size", str(cfg.batch_size), + "--ubatch-size", str(cfg.ubatch_size), + "--parallel", str(cfg.parallel), + "--host", cfg.host, + "--port", str(cfg.container_port), + ] + + if cfg.jinja: + cmd.append("--jinja") + cmd += ["--reasoning", cfg.reasoning] + if cfg.no_context_shift: + cmd.append("--no-context-shift") + if cfg.fa: + cmd += ["-fa", "on"] + if cfg.kv_unified: + cmd.append("--kv-unified") + if cfg.cont_batching: + cmd.append("--cont-batching") + cmd.extend(cfg.extra_args) + return cmd + + +def format_command_for_display(cmd: list) -> str: + return " ".join(shlex.quote(part) for part in cmd) + + +def start_llama_container(cfg: ServerConfig) -> str: + """Remove any pre-existing container of the same name, then start fresh. + + Returns the new container ID. + """ + if container_exists(cfg.container_name): + remove_container(cfg.container_name) + cp = _run(build_run_command(cfg), check=True) + return cp.stdout.strip() + + +# Backwards-compatible alias matching the original spec naming. +start_container = start_llama_container diff --git a/src/llamacppctl/http_ops.py b/src/llamacppctl/http_ops.py new file mode 100644 index 0000000..3fe14e9 --- /dev/null +++ b/src/llamacppctl/http_ops.py @@ -0,0 +1,116 @@ +"""HTTP health checks and chat completions against the llama.cpp OpenAI-compatible API.""" + +from __future__ import annotations + +import time +from typing import Optional + +import requests + +from .schema import PromptConfig, ServerConfig + + +class HttpError(RuntimeError): + pass + + +def base_url(cfg: ServerConfig) -> str: + return f"http://127.0.0.1:{cfg.host_port}/v1" + + +def chat_url(cfg: ServerConfig) -> str: + """Chat completions endpoint, honoring the configurable cfg.chat_endpoint + (consistent with health_endpoint / models_endpoint). Always talks to the + locally published host port on loopback.""" + return f"http://127.0.0.1:{cfg.host_port}{cfg.chat_endpoint}" + + +def check_health(cfg: ServerConfig, timeout: float = 3.0) -> bool: + try: + r = requests.get(f"http://127.0.0.1:{cfg.host_port}{cfg.health_endpoint}", timeout=timeout) + return r.ok + except requests.RequestException: + return False + + +def check_models(cfg: ServerConfig, timeout: float = 3.0) -> bool: + try: + r = requests.get(f"http://127.0.0.1:{cfg.host_port}{cfg.models_endpoint}", timeout=timeout) + return r.ok + except requests.RequestException: + return False + + +def _build_messages(system_prompt: Optional[str], user_prompt: Optional[str]) -> list: + messages = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + if user_prompt: + messages.append({"role": "user", "content": user_prompt}) + if not messages: + messages = [{"role": "user", "content": "ping"}] + return messages + + +def _chat_payload(cfg: ServerConfig, prompt_cfg: PromptConfig) -> dict: + payload = { + "model": cfg.model_alias, + "messages": _build_messages(prompt_cfg.system_prompt, prompt_cfg.user_prompt), + "max_tokens": prompt_cfg.max_tokens, + "stream": prompt_cfg.stream, + } + # Only override temperature when explicitly set; otherwise defer to the + # server's configured --temp so there is a single source of truth. + if prompt_cfg.temperature is not None: + payload["temperature"] = prompt_cfg.temperature + return payload + + +def chat_completion(cfg: ServerConfig, prompt_cfg: PromptConfig) -> requests.Response: + """Low-level chat call returning the raw Response (never raises on HTTP + status). Used by readiness polling, which must treat non-200 as 'not yet'.""" + return requests.post(chat_url(cfg), json=_chat_payload(cfg, prompt_cfg), timeout=10) + + +def chat_completion_text( + cfg: ServerConfig, prompt_cfg: PromptConfig, timeout: float = 30.0 +) -> str: + """High-level chat helper for --chat and the --start/--check post-checks. + + Returns the assistant message content, raising HttpError on transport, + HTTP status, JSON decoding, or malformed-response failures.""" + try: + resp = requests.post(chat_url(cfg), json=_chat_payload(cfg, prompt_cfg), timeout=timeout) + except requests.RequestException as exc: + raise HttpError(f"chat completion request failed: {exc}") from exc + + if resp.status_code != 200: + raise HttpError(f"chat completion returned HTTP {resp.status_code}: {resp.text[:500]}") + + try: + data = resp.json() + except ValueError as exc: + raise HttpError("chat completion response was not valid JSON") from exc + + try: + return data["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError) as exc: + raise HttpError("chat completion response missing choices[0].message.content") from exc + + +def wait_until_ready(cfg: ServerConfig, poll_prompt: PromptConfig) -> bool: + """Poll /v1/chat/completions until it returns HTTP 200 or the timeout elapses. + + A real chat completion is used (not just a port check) because that is + what proves the model is actually loaded and inference-capable. + """ + deadline = time.time() + cfg.timeout + while time.time() < deadline: + try: + r = chat_completion(cfg, poll_prompt) + if r.status_code == 200: + return True + except requests.RequestException: + pass + time.sleep(cfg.poll_interval) + return False diff --git a/src/llamacppctl/lock_ops.py b/src/llamacppctl/lock_ops.py new file mode 100644 index 0000000..dd177d4 --- /dev/null +++ b/src/llamacppctl/lock_ops.py @@ -0,0 +1,37 @@ +"""Exclusive, non-blocking file locking for --start and --change. + +Prevents two concurrent llamacppctl invocations from manipulating the same +container_name at once (mirrors the flock/exit-75 idea from switch-llm.sh). +""" + +from __future__ import annotations + +import fcntl +from pathlib import Path + + +class LockError(RuntimeError): + pass + + +class FileLock: + def __init__(self, path: Path): + self.path = Path(path) + self.fd = None + + def __enter__(self) -> "FileLock": + self.path.parent.mkdir(parents=True, exist_ok=True) + self.fd = open(self.path, "w") + try: + fcntl.flock(self.fd.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + self.fd.close() + self.fd = None + raise LockError(f"lock busy: {self.path}") from exc + return self + + def __exit__(self, exc_type, exc, tb) -> None: + if self.fd: + fcntl.flock(self.fd.fileno(), fcntl.LOCK_UN) + self.fd.close() + self.fd = None diff --git a/src/llamacppctl/main.py b/src/llamacppctl/main.py new file mode 100644 index 0000000..ed759f3 --- /dev/null +++ b/src/llamacppctl/main.py @@ -0,0 +1,104 @@ +"""Thin executable entry point for llamacppctl. + +Orchestration order: + 1. Parse CLI arguments (cli.build_parser) + 2. Run semantic validation (cli.validate_args) + 3. Check that docker is available + 4. Resolve prompt sources (prompt_io) under an InputPolicy built from CLI flags + 5. Resolve effective server/prompt config (config.resolve_effective_config) + 6. Dispatch to the requested action (actions.do_*) +""" + +from __future__ import annotations + +import sys +from typing import Optional, Sequence + +from . import actions +from .cli import build_parser, validate_args +from .config import ConfigError, resolve_effective_config +from .docker_ops import docker_available +from .prompt_io import InputPolicy, PromptSourceError, load_prompt_source, resolve_source + + +def _build_input_policy(args) -> InputPolicy: + return InputPolicy( + max_input_bytes=args.max_input_bytes, + connect_timeout=args.connect_timeout, + read_timeout=args.read_timeout, + allow_insecure_http=args.allow_insecure_http, + allow_private_url=args.allow_private_url, + allow_ip_host=args.allow_ip_host, + follow_redirects=args.follow_redirects, + allow_html_input=args.allow_html_input, + allow_symlinks=args.allow_symlinks, + url_allow_hosts=list(args.url_allow_host or []), + ) + + +def _resolve_prompts(args, policy: InputPolicy) -> None: + """Loads --system*/--prompt* sources and stashes results on args as + _resolved_system_prompt / _resolved_user_prompt for config.py to pick up.""" + system_source = resolve_source(args.system, args.system_file, args.system_url) + prompt_source = resolve_source(args.prompt, args.prompt_file, args.prompt_url) + + args._resolved_system_prompt = ( + load_prompt_source(system_source, policy) if system_source else None + ) + args._resolved_user_prompt = ( + load_prompt_source(prompt_source, policy) if prompt_source else None + ) + + +def run(argv: Optional[Sequence[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + validate_args(args, parser) + + if not docker_available(): + print("docker is not available on PATH (or the daemon is not reachable)", file=sys.stderr) + return 1 + + try: + policy = _build_input_policy(args) + _resolve_prompts(args, policy) + except PromptSourceError as exc: + print(f"prompt input error: {exc}", file=sys.stderr) + return 1 + + try: + server_cfg, prompt_cfg = resolve_effective_config(args) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 1 + + if args.print_effective_config: + actions.print_effective_config(server_cfg, prompt_cfg) + # An action is always required by argparse; only exit early if the run + # is purely diagnostic (no action would actually do anything). + if not (args.start or args.check or args.stop or args.change or args.chat): + return 0 + + if args.start: + return actions.do_start(server_cfg, prompt_cfg, args) + if args.check: + result = actions.do_check(server_cfg, prompt_cfg, args) + actions.print_check_result(server_cfg, result) + return 0 + if args.stop: + return actions.do_stop(server_cfg, args) + if args.change: + return actions.do_change(server_cfg, prompt_cfg, args) + if args.chat: + return actions.do_chat(server_cfg, prompt_cfg, args) + + parser.error("no action specified") + return 2 + + +def main() -> None: + sys.exit(run()) + + +if __name__ == "__main__": + main() diff --git a/src/llamacppctl/prompt_io.py b/src/llamacppctl/prompt_io.py new file mode 100644 index 0000000..ee04ac5 --- /dev/null +++ b/src/llamacppctl/prompt_io.py @@ -0,0 +1,350 @@ +"""Secure prompt input layer. + +Provides exactly three input kinds for --system*/--prompt* CLI options: +literal text, local file, or remote HTTPS(S) URL. + +Security posture (SSRF-hardened, conservative defaults): + - Local files: regular files only, size-limited, UTF-8 text only. + - Remote URLs: https-only by default, no embedded credentials, no + IP-literal hosts by default, DNS resolution + IP classification for + every candidate address, private/loopback/link-local/metadata/reserved + ranges blocked by default, no redirects by default (each hop + re-validated if enabled), strict content-type allowlist, hard size + limit enforced both via Content-Length and while streaming. + +This module intentionally has no knowledge of argparse; it only consumes +a resolved PromptSource plus an InputPolicy. +""" + +from __future__ import annotations + +import contextlib +import ipaddress +import socket +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Optional +from urllib.parse import urljoin, urlparse + +import requests + +SourceKind = Literal["literal", "file", "url"] + +USER_AGENT = "llamacppctl/0.1" + + +class PromptSourceError(ValueError): + """Raised whenever a prompt source is invalid, unsafe, or unreadable.""" + + +@dataclass +class InputPolicy: + max_input_bytes: int = 1_048_576 # 1 MiB + connect_timeout: float = 3.0 + read_timeout: float = 10.0 + + allow_insecure_http: bool = False + allow_private_url: bool = False + allow_ip_host: bool = False + follow_redirects: bool = False + allow_html_input: bool = False + allow_symlinks: bool = False + + url_allow_hosts: list = field(default_factory=list) + + allowed_content_types: set = field( + default_factory=lambda: { + "text/plain", + "text/markdown", + "text/x-markdown", + "text/csv", + "application/json", + "application/xml", + "text/xml", + "application/yaml", + "text/yaml", + "application/x-yaml", + } + ) + + max_redirects: int = 3 + + +@dataclass +class PromptSource: + kind: SourceKind + value: str + + +def resolve_source( + literal: Optional[str], file_: Optional[str], url: Optional[str] +) -> Optional[PromptSource]: + """Translate the (at most one, enforced upstream by argparse) provided + CLI value into a single PromptSource, or None if nothing was given.""" + provided = [v for v in (literal, file_, url) if v is not None] + if len(provided) > 1: + # Defense in depth: argparse mutually-exclusive groups should already + # prevent this, but never trust a single layer for a security rule. + raise PromptSourceError("more than one source provided for the same field") + if literal is not None: + return PromptSource("literal", literal) + if file_ is not None: + return PromptSource("file", file_) + if url is not None: + return PromptSource("url", url) + return None + + +def normalize_text(text: str) -> str: + text = text.replace("\r\n", "\n").replace("\r", "\n") + text = text.lstrip("\ufeff") + if "\x00" in text: + raise PromptSourceError("input contains NUL bytes") + return text + + +def load_text_file(path_str: str, policy: InputPolicy) -> str: + path = Path(path_str).expanduser() + + if not path.exists(): + raise PromptSourceError(f"file not found: {path}") + + if not policy.allow_symlinks and path.is_symlink(): + raise PromptSourceError(f"symlinks are not allowed: {path}") + + if path.is_symlink() and policy.allow_symlinks: + # Allowed, but callers may want to log this; kept silent at this layer. + pass + + if not path.is_file(): + raise PromptSourceError(f"not a regular file: {path}") + + size = path.stat().st_size + if size > policy.max_input_bytes: + raise PromptSourceError( + f"file too large: {size} bytes > {policy.max_input_bytes}" + ) + + raw = path.read_bytes() + + if b"\x00" in raw: + raise PromptSourceError(f"binary input rejected: {path}") + + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + try: + text = raw.decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise PromptSourceError(f"file is not valid UTF-8 text: {path}") from exc + + text = normalize_text(text) + + if len(text.encode("utf-8")) > policy.max_input_bytes: + raise PromptSourceError( + f"decoded text exceeds limit: {policy.max_input_bytes} bytes" + ) + + return text + + +def classify_ip(ip_str: str) -> dict: + ip = ipaddress.ip_address(ip_str) + return { + "is_private": ip.is_private, + "is_loopback": ip.is_loopback, + "is_link_local": ip.is_link_local, + "is_multicast": ip.is_multicast, + "is_reserved": ip.is_reserved, + "is_unspecified": ip.is_unspecified, + } + + +def is_ip_literal(hostname: str) -> bool: + try: + ipaddress.ip_address(hostname) + return True + except ValueError: + return False + + +def resolve_all_ips(hostname: str, port: Optional[int]) -> list: + infos = socket.getaddrinfo(hostname, port, proto=socket.IPPROTO_TCP) + ips = [] + for info in infos: + sockaddr = info[4] + ip = sockaddr[0] + if ip not in ips: + ips.append(ip) + return ips + + +@contextlib.contextmanager +def _pin_dns(host: str, allowed_ips: list): + """Constrain DNS resolution for ``host`` to the already-validated IPs for + the duration of a single request. + + This closes the TOCTOU / DNS-rebinding window between validate_url_target() + (which resolves + classifies) and the actual socket connect (which resolves + again). TLS/SNI and certificate validation stay intact because the request + URL still carries the hostname; only name resolution is restricted. + """ + real_getaddrinfo = socket.getaddrinfo + allowed = set(allowed_ips) + + def pinned(h, *args, **kwargs): + results = real_getaddrinfo(h, *args, **kwargs) + if h == host: + results = [r for r in results if r[4][0] in allowed] + if not results: + raise socket.gaierror( + f"DNS for {host} no longer resolves to a validated address" + ) + return results + + socket.getaddrinfo = pinned + try: + yield + finally: + socket.getaddrinfo = real_getaddrinfo + + +def validate_url_target(url: str, policy: InputPolicy) -> list: + """Validate a URL target and return the list of resolved, allowed IPs.""" + parsed = urlparse(url) + + if parsed.scheme not in ("https", "http"): + raise PromptSourceError(f"unsupported URL scheme: {parsed.scheme!r}") + + if parsed.scheme == "http" and not policy.allow_insecure_http: + raise PromptSourceError("http requires --allow-insecure-http") + + if not parsed.hostname: + raise PromptSourceError("URL has no hostname") + + if parsed.username or parsed.password: + raise PromptSourceError("embedded URL credentials are not allowed") + + host = parsed.hostname.rstrip(".").lower() + + if policy.url_allow_hosts and host not in {h.lower() for h in policy.url_allow_hosts}: + raise PromptSourceError(f"host not in allowlist: {host}") + + if host == "localhost": + raise PromptSourceError("localhost is not allowed") + + if is_ip_literal(host) and not policy.allow_ip_host: + raise PromptSourceError("IP literal hosts require --allow-ip-host") + + ips = resolve_all_ips(host, parsed.port) + if not ips: + raise PromptSourceError(f"hostname did not resolve: {host}") + + if policy.allow_private_url: + return ips + + for ip in ips: + if ip == "169.254.169.254": + raise PromptSourceError("cloud metadata IP is blocked") + flags = classify_ip(ip) + if any(flags.values()): + raise PromptSourceError(f"resolved to blocked IP {ip}: {flags}") + + return ips + + +def is_allowed_content_type(content_type: str, policy: InputPolicy) -> bool: + mime = content_type.split(";", 1)[0].strip().lower() + if mime == "text/html" and policy.allow_html_input: + return True + return mime in policy.allowed_content_types + + +def load_text_url(url: str, policy: InputPolicy, max_redirects: Optional[int] = None) -> str: + """Fetch text from a remote URL with SSRF-hardened validation. + + Every hop (including redirects, if enabled) is independently validated + via validate_url_target() before any request is issued. + """ + if max_redirects is None: + max_redirects = policy.max_redirects + + current = url + + for _ in range(max_redirects + 1): + allowed_ips = validate_url_target(current, policy) + host = (urlparse(current).hostname or "").rstrip(".").lower() + + try: + with _pin_dns(host, allowed_ips): + resp = requests.get( + current, + stream=True, + allow_redirects=False, + timeout=(policy.connect_timeout, policy.read_timeout), + headers={"User-Agent": USER_AGENT}, + ) + except requests.RequestException as exc: + raise PromptSourceError(f"URL fetch failed: {exc}") from exc + + if 300 <= resp.status_code < 400: + if not policy.follow_redirects: + raise PromptSourceError(f"redirect rejected: HTTP {resp.status_code}") + location = resp.headers.get("Location") + if not location: + raise PromptSourceError("redirect without Location header") + current = urljoin(current, location) + continue + + resp.raise_for_status() + + ctype = resp.headers.get("Content-Type", "") + if not is_allowed_content_type(ctype, policy): + raise PromptSourceError(f"disallowed content-type: {ctype!r}") + + clen = resp.headers.get("Content-Length") + if clen is not None: + try: + if int(clen) > policy.max_input_bytes: + raise PromptSourceError( + f"response too large: {clen} bytes > {policy.max_input_bytes}" + ) + except ValueError: + pass + + raw = bytearray() + for chunk in resp.iter_content(chunk_size=8192): + if not chunk: + continue + raw.extend(chunk) + if len(raw) > policy.max_input_bytes: + raise PromptSourceError( + f"response exceeded max size: {len(raw)} bytes > {policy.max_input_bytes}" + ) + + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise PromptSourceError("remote content is not valid UTF-8 text") from exc + + return normalize_text(text) + + raise PromptSourceError(f"too many redirects (>{max_redirects})") + + +def load_prompt_source(source: PromptSource, policy: InputPolicy) -> str: + if source.kind == "literal": + text = normalize_text(source.value) + elif source.kind == "file": + text = load_text_file(source.value, policy) + elif source.kind == "url": + text = load_text_url(source.value, policy) + else: + raise PromptSourceError(f"unsupported source kind: {source.kind}") + + if len(text.encode("utf-8")) > policy.max_input_bytes: + raise PromptSourceError( + f"normalized text exceeds max size: {policy.max_input_bytes}" + ) + + return text diff --git a/src/llamacppctl/py.typed b/src/llamacppctl/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/llamacppctl/schema.py b/src/llamacppctl/schema.py new file mode 100644 index 0000000..e6d6209 --- /dev/null +++ b/src/llamacppctl/schema.py @@ -0,0 +1,79 @@ +"""Core data models shared across config resolution, docker ops, and http ops.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Literal, Optional + +ReasoningMode = Literal["on", "off"] + + +@dataclass +class ServerConfig: + """Everything needed to build and run the llama.cpp Docker container.""" + + image: str + hf_home: Path + model_path: str + container_name: str + host_port: int + container_port: int + model_alias: str + gpu_device: str + restart_policy: str + + ctx_size: int + n_predict: int + temp: float + top_p: float + top_k: int + min_p: float + repeat_penalty: float + main_gpu: int + ngl: int + batch_size: int + ubatch_size: int + parallel: int + + cache_type_k: str + cache_type_v: str + reasoning: ReasoningMode + jinja: bool + fa: bool + kv_unified: bool + cont_batching: bool + no_context_shift: bool + + host: str + health_endpoint: str + models_endpoint: str + chat_endpoint: str + timeout: int + poll_interval: float + lock_file: Path + + extra_args: list = field(default_factory=list) + + +@dataclass +class PromptConfig: + system_prompt: Optional[str] = None + user_prompt: Optional[str] = None + # Reasoning models spend many tokens "thinking" before emitting the final + # answer; a small budget truncates mid-reasoning and yields empty content. + max_tokens: int = 2048 + # None => omit from the request so the server's configured --temp applies. + temperature: Optional[float] = None + stream: bool = False + + +@dataclass +class CheckResult: + container_exists: bool + container_running: bool + http_ok: bool + models_ok: bool + chat_ok: bool + base_url: str + detail: str = "" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..fd68f14 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,18 @@ +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from llamacppctl.prompt_io import InputPolicy # noqa: E402 + + +@pytest.fixture +def policy() -> InputPolicy: + return InputPolicy() + + +@pytest.fixture +def small_policy() -> InputPolicy: + return InputPolicy(max_input_bytes=16) diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..4d9c7e7 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,145 @@ +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from llamacppctl.cli import build_parser, validate_args # noqa: E402 + + +def parse(argv): + parser = build_parser() + args = parser.parse_args(argv) + validate_args(args, parser) + return args + + +def parse_error(argv): + parser = build_parser() + with pytest.raises(SystemExit): + args = parser.parse_args(argv) + validate_args(args, parser) + + +def test_requires_one_action(): + parse_error([]) + + +def test_actions_are_mutually_exclusive(): + parse_error(["--start", "--stop"]) + + +def test_start_is_valid_minimal(): + args = parse(["--start"]) + assert args.start is True + + +def test_chat_requires_prompt_source(): + parse_error(["--chat"]) + + +def test_chat_with_prompt_is_valid(): + args = parse(["--chat", "-p", "hello"]) + assert args.chat is True + assert args.prompt == "hello" + + +def test_stop_rejects_prompt_sources(): + parse_error(["--stop", "-p", "hello"]) + + +def test_stop_rejects_system_sources(): + parse_error(["--stop", "-s", "system text"]) + + +def test_check_with_system_but_no_prompt_rejected(): + parse_error(["--check", "-s", "system text"]) + + +def test_check_with_system_and_prompt_ok(): + args = parse(["--check", "-s", "system text", "-p", "hello"]) + assert args.check is True + + +def test_mutually_exclusive_system_sources(): + parse_error(["--start", "-s", "a", "--system-file", "b.txt"]) + + +def test_mutually_exclusive_prompt_sources(): + parse_error(["--start", "-p", "a", "--prompt-url", "https://example.com/p"]) + + +def test_max_input_bytes_must_be_positive(): + parse_error(["--start", "--max-input-bytes", "0"]) + + +def test_negative_max_input_bytes_rejected(): + parse_error(["--start", "--max-input-bytes", "-5"]) + + +def test_connect_timeout_must_be_positive(): + parse_error(["--start", "--connect-timeout", "0"]) + + +def test_read_timeout_must_be_positive(): + parse_error(["--start", "--read-timeout", "-1"]) + + +def test_allow_private_url_conflicts_with_allowlist(): + parse_error(["--start", "--allow-private-url", "--url-allow-host", "example.com"]) + + +def test_host_port_out_of_range_rejected(): + parse_error(["--start", "--host-port", "70000"]) + + +def test_host_port_zero_rejected(): + parse_error(["--start", "--host-port", "0"]) + + +def test_container_port_out_of_range_rejected(): + parse_error(["--start", "--container-port", "-1"]) + + +def test_container_name_empty_string_rejected(): + parse_error(["--start", "--container-name", " "]) + + +def test_container_name_valid(): + args = parse(["--start", "--container-name", "my_llm"]) + assert args.container_name == "my_llm" + + +def test_boolean_flag_pairs_default_none(): + args = parse(["--start"]) + assert args.jinja is None + assert args.fa is None + assert args.kv_unified is None + assert args.cont_batching is None + assert args.no_context_shift is None + + +def test_boolean_flag_pairs_explicit_true_false(): + args = parse(["--start", "--jinja", "--no-fa"]) + assert args.jinja is True + assert args.fa is False + + +def test_dry_run_flag(): + args = parse(["--start", "--dry-run"]) + assert args.dry_run is True + + +def test_print_effective_config_flag(): + args = parse(["--start", "--print-effective-config"]) + assert args.print_effective_config is True + + +def test_reasoning_choice_validated(): + parse_error(["--start", "--reasoning", "maybe"]) + + +def test_reasoning_choice_valid(): + args = parse(["--start", "--reasoning", "off"]) + assert args.reasoning == "off" diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..7ed12f6 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,220 @@ +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from llamacppctl.config import ConfigError, resolve_effective_config, builtin_defaults # noqa: E402 +from llamacppctl.cli import build_parser, validate_args # noqa: E402 + + +CONFIG_BASIC = """ +[default] +image = ghcr.io/ggml-org/llama.cpp:server-cuda +hf_home = /srv/models +model_path = qwen3/default.gguf +container_name = test_default +host_port = 8001 +container_port = 8000 +model_alias = default_llm +gpu_device = 0 +restart_policy = unless-stopped +ctx_size = 262144 +n_predict = 16384 +temp = 0.65 +top_p = 0.80 +top_k = 20 +min_p = 0.01 +repeat_penalty = 1.05 +main_gpu = 0 +ngl = 999 +fa = true +kv_unified = true +jinja = true +reasoning = on +no_context_shift = true +cache_type_k = q4_0 +cache_type_v = q4_0 +batch_size = 1024 +ubatch_size = 512 +parallel = 1 +cont_batching = true +host = 0.0.0.0 +health_endpoint = /health +models_endpoint = /v1/models +chat_endpoint = /v1/chat/completions +timeout = 300 +poll_interval = 2 + +[model.alt] +container_name = test_alt +host_port = 9001 +model_alias = alt_llm + +[model.noname] +host_port = 9002 + +[prompt.concise] +system_prompt = Be brief. +""" + + +def _write_config(tmp_path: Path, content: str = CONFIG_BASIC) -> Path: + p = tmp_path / "llama.cpp.config" + p.write_text(content, encoding="utf-8") + return p + + +def _parse(argv): + parser = build_parser() + args = parser.parse_args(argv) + validate_args(args, parser) + args._resolved_system_prompt = None + args._resolved_user_prompt = None + return args + + +def test_default_profile_resolves(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse(["--start", "--config", str(cfg_path)]) + server_cfg, prompt_cfg = resolve_effective_config(args) + assert server_cfg.container_name == "test_default" + assert server_cfg.host_port == 8001 + assert server_cfg.jinja is True + assert server_cfg.reasoning == "on" + + +def test_model_profile_overrides_default(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse(["--start", "--config", str(cfg_path), "--profile", "alt"]) + server_cfg, _ = resolve_effective_config(args) + assert server_cfg.container_name == "test_alt" + assert server_cfg.host_port == 9001 + assert server_cfg.model_alias == "alt_llm" + # inherited from [default] + assert server_cfg.image == "ghcr.io/ggml-org/llama.cpp:server-cuda" + + +def test_missing_profile_raises(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse(["--start", "--config", str(cfg_path), "--profile", "does-not-exist"]) + with pytest.raises(ConfigError): + resolve_effective_config(args) + + +def test_prompt_profile_provides_fallback_system_prompt(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse( + ["--start", "--config", str(cfg_path), "--system-prompt-profile", "concise"] + ) + _, prompt_cfg = resolve_effective_config(args) + assert prompt_cfg.system_prompt == "Be brief." + + +def test_explicit_system_overrides_prompt_profile(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse( + [ + "--start", + "--config", + str(cfg_path), + "--system-prompt-profile", + "concise", + "--system", + "Explicit wins.", + ] + ) + args._resolved_system_prompt = "Explicit wins." + _, prompt_cfg = resolve_effective_config(args) + assert prompt_cfg.system_prompt == "Explicit wins." + + +def test_missing_container_name_raises(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse(["--start", "--config", str(cfg_path), "--profile", "noname"]) + # [model.noname] has no container_name and [default] provides one, + # so this should actually succeed by inheriting test_default. + server_cfg, _ = resolve_effective_config(args) + assert server_cfg.container_name == "test_default" + + +def test_empty_container_name_via_cli_rejected_by_argparse(tmp_path): + # cli.validate_args() already rejects a blank --container-name before + # config resolution is ever reached (defense at the CLI layer). + cfg_path = _write_config(tmp_path) + with pytest.raises(SystemExit): + _parse(["--start", "--config", str(cfg_path), "--container-name", " "]) + + +def test_empty_container_name_from_config_raises(tmp_path): + # container_name can only end up empty via the config file itself + # (CLI-level blanks are already rejected by validate_args above). + bad_config = CONFIG_BASIC.replace("container_name = test_default", "container_name =") + cfg_path = _write_config(tmp_path, bad_config) + args = _parse(["--start", "--config", str(cfg_path)]) + with pytest.raises(ConfigError): + resolve_effective_config(args) + + +def test_cli_override_wins_over_config(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse( + ["--start", "--config", str(cfg_path), "--host-port", "12345"] + ) + server_cfg, _ = resolve_effective_config(args) + assert server_cfg.host_port == 12345 + + +def test_invalid_boolean_raises(tmp_path): + bad_config = CONFIG_BASIC.replace("jinja = true", "jinja = maybe") + cfg_path = _write_config(tmp_path, bad_config) + args = _parse(["--start", "--config", str(cfg_path)]) + with pytest.raises(ConfigError): + resolve_effective_config(args) + + +def test_lock_file_derived_from_container_name(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse(["--start", "--config", str(cfg_path)]) + server_cfg, _ = resolve_effective_config(args) + assert str(server_cfg.lock_file) == "/tmp/llamacppctl.test_default.lock" + + +def test_builtin_defaults_has_container_name(): + defaults = builtin_defaults() + assert defaults["container_name"] == "va_llm" + + +def test_chat_params_default_and_defer_temperature(tmp_path): + cfg_path = _write_config(tmp_path) + args = _parse(["--start", "--config", str(cfg_path)]) + _, prompt_cfg = resolve_effective_config(args) + assert prompt_cfg.max_tokens == 2048 # built-in default + assert prompt_cfg.temperature is None # empty chat_temperature => server temp + + +def test_chat_params_from_config(tmp_path): + cfg = CONFIG_BASIC.replace( + "poll_interval = 2", + "poll_interval = 2\nmax_tokens = 32768\nchat_temperature = 0.6", + ) + cfg_path = _write_config(tmp_path, cfg) + args = _parse(["--start", "--config", str(cfg_path)]) + _, prompt_cfg = resolve_effective_config(args) + assert prompt_cfg.max_tokens == 32768 + assert prompt_cfg.temperature == 0.6 + + +def test_chat_params_cli_overrides_config(tmp_path): + cfg = CONFIG_BASIC.replace( + "poll_interval = 2", + "poll_interval = 2\nmax_tokens = 32768\nchat_temperature = 0.6", + ) + cfg_path = _write_config(tmp_path, cfg) + args = _parse( + ["--start", "--config", str(cfg_path), "--max-tokens", "512", "--chat-temp", "0.1"] + ) + _, prompt_cfg = resolve_effective_config(args) + assert prompt_cfg.max_tokens == 512 + assert prompt_cfg.temperature == 0.1 diff --git a/tests/test_docker_ops.py b/tests/test_docker_ops.py new file mode 100644 index 0000000..6ceab97 --- /dev/null +++ b/tests/test_docker_ops.py @@ -0,0 +1,154 @@ +import sys +from pathlib import Path +from unittest.mock import patch, MagicMock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from llamacppctl.docker_ops import ( # noqa: E402 + build_run_command, + container_exists, + container_running, + docker_available, + format_command_for_display, + inspect_container, +) +from llamacppctl.schema import ServerConfig # noqa: E402 + + +def make_cfg(**overrides) -> ServerConfig: + base = dict( + image="ghcr.io/ggml-org/llama.cpp:server-cuda", + hf_home=Path("/srv/models"), + model_path="qwen3/default.gguf", + container_name="test_llm", + host_port=8001, + container_port=8000, + model_alias="default_llm", + gpu_device="0", + restart_policy="unless-stopped", + ctx_size=262144, + n_predict=16384, + temp=0.65, + top_p=0.80, + top_k=20, + min_p=0.01, + repeat_penalty=1.05, + main_gpu=0, + ngl=999, + batch_size=1024, + ubatch_size=512, + parallel=1, + cache_type_k="q4_0", + cache_type_v="q4_0", + reasoning="on", + jinja=True, + fa=True, + kv_unified=True, + cont_batching=True, + no_context_shift=True, + host="0.0.0.0", + health_endpoint="/health", + models_endpoint="/v1/models", + chat_endpoint="/v1/chat/completions", + timeout=300, + poll_interval=2.0, + lock_file=Path("/tmp/llamacppctl.test_llm.lock"), + ) + base.update(overrides) + return ServerConfig(**base) + + +def test_docker_available_true(): + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0) + assert docker_available() is True + + +def test_docker_available_false_when_missing(): + with patch("subprocess.run", side_effect=FileNotFoundError()): + assert docker_available() is False + + +def test_container_exists_true(): + with patch("llamacppctl.docker_ops._run") as mock_run: + mock_run.return_value = MagicMock(stdout="test_llm\nother\n") + assert container_exists("test_llm") is True + + +def test_container_exists_false(): + with patch("llamacppctl.docker_ops._run") as mock_run: + mock_run.return_value = MagicMock(stdout="other\n") + assert container_exists("test_llm") is False + + +def test_container_running_true(): + with patch("llamacppctl.docker_ops._run") as mock_run: + mock_run.return_value = MagicMock(stdout="test_llm\n") + assert container_running("test_llm") is True + + +def test_inspect_container_missing(): + with patch("llamacppctl.docker_ops.container_exists", return_value=False): + info = inspect_container("ghost") + assert info.status == "missing" + assert info.health == "none" + + +def test_inspect_container_running(): + fake_json = ( + '[{"State": {"Status": "running", "Health": {"Status": "healthy"}}}]' + ) + with patch("llamacppctl.docker_ops.container_exists", return_value=True), patch( + "llamacppctl.docker_ops._run" + ) as mock_run: + mock_run.return_value = MagicMock(stdout=fake_json) + info = inspect_container("test_llm") + assert info.status == "running" + assert info.health == "healthy" + + +def test_build_run_command_contains_key_flags(): + cfg = make_cfg() + cmd = build_run_command(cfg) + assert cmd[0:2] == ["docker", "run"] + assert "--gpus" in cmd + assert "device=0" in cmd + assert "--name" in cmd + assert "test_llm" in cmd + assert "-p" in cmd + assert "8001:8000" in cmd + assert "--jinja" in cmd + assert "--reasoning" in cmd + assert "on" in cmd + assert "--no-context-shift" in cmd + assert "--kv-unified" in cmd + assert "--cont-batching" in cmd + + +def test_build_run_command_respects_disabled_flags(): + cfg = make_cfg(jinja=False, kv_unified=False, cont_batching=False, no_context_shift=False, fa=False) + cmd = build_run_command(cfg) + assert "--jinja" not in cmd + assert "--kv-unified" not in cmd + assert "--cont-batching" not in cmd + assert "--no-context-shift" not in cmd + + +def test_build_run_command_absolute_model_path(): + cfg = make_cfg(model_path="/abs/path/model.gguf") + cmd = build_run_command(cfg) + idx = cmd.index("-m") + assert cmd[idx + 1] == "/abs/path/model.gguf" + + +def test_build_run_command_relative_model_path(): + cfg = make_cfg(model_path="qwen3/default.gguf") + cmd = build_run_command(cfg) + idx = cmd.index("-m") + assert cmd[idx + 1] == "/hf_home/qwen3/default.gguf" + + +def test_format_command_for_display_quotes_properly(): + cmd = ["docker", "run", "--name", "has space"] + out = format_command_for_display(cmd) + assert "'has space'" in out diff --git a/tests/test_http_ops.py b/tests/test_http_ops.py new file mode 100644 index 0000000..185dffb --- /dev/null +++ b/tests/test_http_ops.py @@ -0,0 +1,132 @@ +import sys +from pathlib import Path +from unittest.mock import patch, MagicMock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +import requests # noqa: E402 + +from llamacppctl.http_ops import ( # noqa: E402 + HttpError, + base_url, + chat_completion, + chat_completion_text, + check_health, + check_models, + wait_until_ready, +) +from llamacppctl.schema import PromptConfig, ServerConfig # noqa: E402 +from tests.test_docker_ops import make_cfg # noqa: E402 + + +def test_base_url(): + cfg = make_cfg(host_port=8001) + assert base_url(cfg) == "http://127.0.0.1:8001/v1" + + +def test_check_health_ok(): + cfg = make_cfg() + with patch("requests.get") as mock_get: + mock_get.return_value = MagicMock(ok=True) + assert check_health(cfg) is True + + +def test_check_health_exception_returns_false(): + cfg = make_cfg() + with patch("requests.get", side_effect=requests.RequestException("boom")): + assert check_health(cfg) is False + + +def test_check_models_ok(): + cfg = make_cfg() + with patch("requests.get") as mock_get: + mock_get.return_value = MagicMock(ok=True) + assert check_models(cfg) is True + + +def test_chat_completion_posts_expected_payload(): + cfg = make_cfg() + prompt_cfg = PromptConfig(system_prompt="sys", user_prompt="hi", max_tokens=32, temperature=0.1) + with patch("requests.post") as mock_post: + mock_post.return_value = MagicMock(status_code=200) + chat_completion(cfg, prompt_cfg) + _, kwargs = mock_post.call_args + payload = kwargs["json"] + assert payload["messages"][0] == {"role": "system", "content": "sys"} + assert payload["messages"][1] == {"role": "user", "content": "hi"} + assert payload["max_tokens"] == 32 + + +def test_chat_completion_text_success(): + cfg = make_cfg() + prompt_cfg = PromptConfig(user_prompt="hi") + with patch("requests.post") as mock_post: + mock_resp = MagicMock(status_code=200) + mock_resp.json.return_value = {"choices": [{"message": {"content": "ok"}}]} + mock_post.return_value = mock_resp + assert chat_completion_text(cfg, prompt_cfg) == "ok" + + +def test_chat_completion_text_honors_chat_endpoint(): + cfg = make_cfg(host_port=8001, chat_endpoint="/custom/chat") + prompt_cfg = PromptConfig(user_prompt="hi") + with patch("requests.post") as mock_post: + mock_resp = MagicMock(status_code=200) + mock_resp.json.return_value = {"choices": [{"message": {"content": "ok"}}]} + mock_post.return_value = mock_resp + chat_completion_text(cfg, prompt_cfg) + args, _ = mock_post.call_args + assert args[0] == "http://127.0.0.1:8001/custom/chat" + + +def test_chat_completion_text_http_error(): + cfg = make_cfg() + prompt_cfg = PromptConfig(user_prompt="hi") + with patch("requests.post") as mock_post: + mock_post.return_value = MagicMock(status_code=500, text="server error") + try: + chat_completion_text(cfg, prompt_cfg) + assert False, "expected HttpError" + except HttpError: + pass + + +def test_chat_completion_text_transport_error(): + cfg = make_cfg() + prompt_cfg = PromptConfig(user_prompt="hi") + with patch("requests.post", side_effect=requests.RequestException("down")): + try: + chat_completion_text(cfg, prompt_cfg) + assert False, "expected HttpError" + except HttpError: + pass + + +def test_chat_completion_text_malformed_response(): + cfg = make_cfg() + prompt_cfg = PromptConfig(user_prompt="hi") + with patch("requests.post") as mock_post: + mock_resp = MagicMock(status_code=200) + mock_resp.json.return_value = {"unexpected": True} + mock_post.return_value = mock_resp + try: + chat_completion_text(cfg, prompt_cfg) + assert False, "expected HttpError" + except HttpError: + pass + + +def test_wait_until_ready_succeeds_immediately(): + cfg = make_cfg(timeout=5, poll_interval=0.01) + probe = PromptConfig(user_prompt="ping", max_tokens=1) + with patch("llamacppctl.http_ops.chat_completion") as mock_chat: + mock_chat.return_value = MagicMock(status_code=200) + assert wait_until_ready(cfg, probe) is True + + +def test_wait_until_ready_times_out(): + cfg = make_cfg(timeout=0, poll_interval=0.01) + probe = PromptConfig(user_prompt="ping", max_tokens=1) + with patch("llamacppctl.http_ops.chat_completion") as mock_chat: + mock_chat.return_value = MagicMock(status_code=500) + assert wait_until_ready(cfg, probe) is False diff --git a/tests/test_lock_ops.py b/tests/test_lock_ops.py new file mode 100644 index 0000000..44e1884 --- /dev/null +++ b/tests/test_lock_ops.py @@ -0,0 +1,31 @@ +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from llamacppctl.lock_ops import FileLock, LockError # noqa: E402 + + +def test_lock_acquire_and_release(tmp_path): + lock_path = tmp_path / "test.lock" + with FileLock(lock_path): + assert lock_path.exists() + # released cleanly, can acquire again + with FileLock(lock_path): + pass + + +def test_lock_busy_raises(tmp_path): + lock_path = tmp_path / "busy.lock" + with FileLock(lock_path): + with pytest.raises(LockError): + with FileLock(lock_path): + pass + + +def test_lock_creates_parent_dirs(tmp_path): + lock_path = tmp_path / "nested" / "dir" / "test.lock" + with FileLock(lock_path): + assert lock_path.exists() diff --git a/tests/test_prompt_files.py b/tests/test_prompt_files.py new file mode 100644 index 0000000..f693a5b --- /dev/null +++ b/tests/test_prompt_files.py @@ -0,0 +1,81 @@ +import os + +import pytest + +from llamacppctl.prompt_io import PromptSourceError, load_text_file + + +def test_load_utf8_file(tmp_path, policy): + p = tmp_path / "sys.txt" + p.write_text("Du bist präzise.", encoding="utf-8") + assert load_text_file(str(p), policy) == "Du bist präzise." + + +def test_strip_bom(tmp_path, policy): + p = tmp_path / "bom.txt" + p.write_bytes(b"\xef\xbb\xbfhello") + assert load_text_file(str(p), policy) == "hello" + + +def test_normalize_crlf(tmp_path, policy): + p = tmp_path / "crlf.txt" + p.write_bytes(b"line1\r\nline2\r\n") + assert load_text_file(str(p), policy) == "line1\nline2\n" + + +def test_reject_missing_file(tmp_path, policy): + with pytest.raises(PromptSourceError): + load_text_file(str(tmp_path / "nope.txt"), policy) + + +def test_reject_directory(tmp_path, policy): + d = tmp_path / "adir" + d.mkdir() + with pytest.raises(PromptSourceError): + load_text_file(str(d), policy) + + +def test_reject_binary_file(tmp_path, policy): + p = tmp_path / "bin.dat" + p.write_bytes(b"\x00\x01\x02binary") + with pytest.raises(PromptSourceError): + load_text_file(str(p), policy) + + +def test_reject_too_large_file(tmp_path, small_policy): + p = tmp_path / "big.txt" + p.write_text("x" * 100, encoding="utf-8") + with pytest.raises(PromptSourceError): + load_text_file(str(p), small_policy) + + +def test_reject_invalid_utf8(tmp_path, policy): + p = tmp_path / "invalid.txt" + p.write_bytes(b"\xff\xfe\xfa\xfb\x80\x81") + with pytest.raises(PromptSourceError): + load_text_file(str(p), policy) + + +def test_reject_symlink_when_disabled(tmp_path, policy): + target = tmp_path / "target.txt" + target.write_text("hi", encoding="utf-8") + link = tmp_path / "link.txt" + os.symlink(target, link) + + from dataclasses import replace + + strict_policy = replace(policy, allow_symlinks=False) + with pytest.raises(PromptSourceError): + load_text_file(str(link), strict_policy) + + +def test_allow_symlink_when_enabled(tmp_path, policy): + target = tmp_path / "target2.txt" + target.write_text("hi again", encoding="utf-8") + link = tmp_path / "link2.txt" + os.symlink(target, link) + + from dataclasses import replace + + permissive_policy = replace(policy, allow_symlinks=True) + assert load_text_file(str(link), permissive_policy) == "hi again" diff --git a/tests/test_prompt_sources.py b/tests/test_prompt_sources.py new file mode 100644 index 0000000..dd2c26e --- /dev/null +++ b/tests/test_prompt_sources.py @@ -0,0 +1,28 @@ +from llamacppctl.prompt_io import PromptSource, PromptSourceError, resolve_source + + +def test_resolve_literal_source(): + src = resolve_source("hello", None, None) + assert src == PromptSource("literal", "hello") + + +def test_resolve_file_source(): + src = resolve_source(None, "/tmp/foo.txt", None) + assert src == PromptSource("file", "/tmp/foo.txt") + + +def test_resolve_url_source(): + src = resolve_source(None, None, "https://example.org/x.txt") + assert src == PromptSource("url", "https://example.org/x.txt") + + +def test_resolve_none(): + assert resolve_source(None, None, None) is None + + +def test_resolve_rejects_multiple_sources(): + try: + resolve_source("a", "b", None) + assert False, "expected PromptSourceError" + except PromptSourceError: + pass diff --git a/tests/test_prompt_urls.py b/tests/test_prompt_urls.py new file mode 100644 index 0000000..5cc800e --- /dev/null +++ b/tests/test_prompt_urls.py @@ -0,0 +1,203 @@ +import socket +from dataclasses import replace + +import pytest + +from llamacppctl.prompt_io import ( + PromptSourceError, + load_text_url, + validate_url_target, +) + + +class FakeResponse: + def __init__(self, status_code=200, headers=None, chunks=None, text=""): + self.status_code = status_code + self.headers = headers or {} + self._chunks = chunks if chunks is not None else [text.encode("utf-8")] + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + def iter_content(self, chunk_size=8192): + for c in self._chunks: + yield c + + +def _fake_getaddrinfo_factory(mapping): + def fake_getaddrinfo(host, port, *args, **kwargs): + ips = mapping.get(host, []) + return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port or 0)) for ip in ips] + + return fake_getaddrinfo + + +def test_reject_http_without_opt_in(policy): + with pytest.raises(PromptSourceError): + validate_url_target("http://example.org/x.txt", policy) + + +def test_reject_localhost(policy): + with pytest.raises(PromptSourceError): + validate_url_target("https://localhost/x.txt", policy) + + +def test_reject_ip_literal_without_opt_in(policy): + with pytest.raises(PromptSourceError): + validate_url_target("https://93.184.216.34/x.txt", policy) + + +def test_allow_ip_literal_with_opt_in(monkeypatch, policy): + strict_policy = replace(policy, allow_ip_host=True) + monkeypatch.setattr( + socket, "getaddrinfo", _fake_getaddrinfo_factory({"93.184.216.34": ["93.184.216.34"]}) + ) + validate_url_target("https://93.184.216.34/x.txt", strict_policy) + + +def test_reject_private_ip_resolution(monkeypatch, policy): + monkeypatch.setattr( + socket, "getaddrinfo", _fake_getaddrinfo_factory({"internal.example.org": ["10.0.0.5"]}) + ) + with pytest.raises(PromptSourceError): + validate_url_target("https://internal.example.org/x.txt", policy) + + +def test_reject_metadata_ip(monkeypatch, policy): + monkeypatch.setattr( + socket, "getaddrinfo", _fake_getaddrinfo_factory({"evil.example.org": ["169.254.169.254"]}) + ) + with pytest.raises(PromptSourceError): + validate_url_target("https://evil.example.org/x.txt", policy) + + +def test_reject_embedded_credentials(policy): + with pytest.raises(PromptSourceError): + validate_url_target("https://user:pass@example.org/x.txt", policy) + + +def test_allowlist_blocks_other_hosts(monkeypatch, policy): + strict_policy = replace(policy, url_allow_hosts=["good.example.org"]) + monkeypatch.setattr( + socket, + "getaddrinfo", + _fake_getaddrinfo_factory({"bad.example.org": ["93.184.216.34"]}), + ) + with pytest.raises(PromptSourceError): + validate_url_target("https://bad.example.org/x.txt", strict_policy) + + +def test_accept_https_text_plain(monkeypatch, policy): + monkeypatch.setattr( + socket, "getaddrinfo", _fake_getaddrinfo_factory({"example.org": ["93.184.216.34"]}) + ) + + def fake_get(url, **kwargs): + return FakeResponse( + status_code=200, + headers={"Content-Type": "text/plain", "Content-Length": "5"}, + text="hello", + ) + + import llamacppctl.prompt_io as prompt_io + + monkeypatch.setattr(prompt_io.requests, "get", fake_get) + text = load_text_url("https://example.org/x.txt", policy) + assert text == "hello" + + +def test_reject_disallowed_content_type(monkeypatch, policy): + monkeypatch.setattr( + socket, "getaddrinfo", _fake_getaddrinfo_factory({"example.org": ["93.184.216.34"]}) + ) + + def fake_get(url, **kwargs): + return FakeResponse(status_code=200, headers={"Content-Type": "text/html"}, text="

hi

") + + import llamacppctl.prompt_io as prompt_io + + monkeypatch.setattr(prompt_io.requests, "get", fake_get) + with pytest.raises(PromptSourceError): + load_text_url("https://example.org/x.txt", policy) + + +def test_reject_large_content_length(monkeypatch, small_policy): + monkeypatch.setattr( + socket, "getaddrinfo", _fake_getaddrinfo_factory({"example.org": ["93.184.216.34"]}) + ) + + def fake_get(url, **kwargs): + return FakeResponse( + status_code=200, + headers={"Content-Type": "text/plain", "Content-Length": "1000"}, + text="x" * 1000, + ) + + import llamacppctl.prompt_io as prompt_io + + monkeypatch.setattr(prompt_io.requests, "get", fake_get) + with pytest.raises(PromptSourceError): + load_text_url("https://example.org/x.txt", small_policy) + + +def test_reject_large_streamed_response(monkeypatch, small_policy): + monkeypatch.setattr( + socket, "getaddrinfo", _fake_getaddrinfo_factory({"example.org": ["93.184.216.34"]}) + ) + + def fake_get(url, **kwargs): + return FakeResponse( + status_code=200, + headers={"Content-Type": "text/plain"}, + chunks=[b"x" * 10, b"x" * 10, b"x" * 10], + ) + + import llamacppctl.prompt_io as prompt_io + + monkeypatch.setattr(prompt_io.requests, "get", fake_get) + with pytest.raises(PromptSourceError): + load_text_url("https://example.org/x.txt", small_policy) + + +def test_reject_redirect_by_default(monkeypatch, policy): + monkeypatch.setattr( + socket, "getaddrinfo", _fake_getaddrinfo_factory({"example.org": ["93.184.216.34"]}) + ) + + def fake_get(url, **kwargs): + return FakeResponse(status_code=302, headers={"Location": "https://example.org/other.txt"}) + + import llamacppctl.prompt_io as prompt_io + + monkeypatch.setattr(prompt_io.requests, "get", fake_get) + with pytest.raises(PromptSourceError): + load_text_url("https://example.org/x.txt", policy) + + +def test_revalidate_redirect_target(monkeypatch, policy): + strict_policy = replace(policy, follow_redirects=True) + monkeypatch.setattr( + socket, + "getaddrinfo", + _fake_getaddrinfo_factory( + {"example.org": ["93.184.216.34"], "internal.example.org": ["10.0.0.9"]} + ), + ) + + calls = {"n": 0} + + def fake_get(url, **kwargs): + calls["n"] += 1 + if "internal" not in url: + return FakeResponse( + status_code=302, headers={"Location": "https://internal.example.org/secret.txt"} + ) + return FakeResponse(status_code=200, headers={"Content-Type": "text/plain"}, text="leak") + + import llamacppctl.prompt_io as prompt_io + + monkeypatch.setattr(prompt_io.requests, "get", fake_get) + with pytest.raises(PromptSourceError): + load_text_url("https://example.org/x.txt", strict_policy) + assert calls["n"] == 1 # must fail validation before the second request