llamacppctl/build_archive.py

#!/usr/bin/env python3
"""build_archive.py

Assembles ``llamacppctl-installable.tar.gz`` directly on the local machine
from the llamacppctl project files, and verifies that all required files and
declared dependencies are present and consistent before/after packaging.

This script is intentionally self-contained (standard library only, plus an
optional import of ``tomllib``/``tomli`` purely for dependency verification)
so it can be run directly, without first installing the llamacppctl package
itself.

Usage
-----
    python3 build_archive.py
    python3 build_archive.py --project-dir /path/to/llamacppctl --output /path/to/llamacppctl-installable.tar.gz
    python3 build_archive.py --skip-tests          # skip running pytest before packaging
    python3 build_archive.py --skip-pip-check      # skip the pip dependency dry-run

Exit codes
----------
    0  archive built and verified successfully
    1  a required file was missing, a test failed, or verification failed
"""

from __future__ import annotations

import argparse
import subprocess
import sys
import tarfile
import venv
from pathlib import Path

# --------------------------------------------------------------------------
# Required file manifest.
#
# Every path here is relative to the project directory and MUST exist before
# packaging. This is the "verifies that all dependencies are correctly
# included" contract: the archive is only built if every one of these is
# present, and re-opened afterwards to confirm every one of them made it
# into the tarball unmodified.
# --------------------------------------------------------------------------
REQUIRED_FILES = [
    "pyproject.toml",
    "README.md",
    "LICENSE",
    "CHANGELOG.md",
    "requirements.txt",
    "requirements-dev.txt",
    "llama.cpp.config.example",
    "docs/SECURITY_AND_OPERATIONS.md",
    "man/llamacppctl.1",
    "src/llamacppctl/__init__.py",
    "src/llamacppctl/py.typed",
    "src/llamacppctl/cli.py",
    "src/llamacppctl/schema.py",
    "src/llamacppctl/config.py",
    "src/llamacppctl/prompt_io.py",
    "src/llamacppctl/docker_ops.py",
    "src/llamacppctl/http_ops.py",
    "src/llamacppctl/lock_ops.py",
    "src/llamacppctl/actions.py",
    "src/llamacppctl/main.py",
    "tests/conftest.py",
    "tests/test_prompt_sources.py",
    "tests/test_prompt_files.py",
    "tests/test_prompt_urls.py",
    "tests/test_config.py",
    "tests/test_cli.py",
    "tests/test_docker_ops.py",
    "tests/test_http_ops.py",
    "tests/test_lock_ops.py",
    "tests/test_actions.py",
    "tests/test_build_archive.py",
    "scripts/smoke.sh",
    "docs/BEDIENUNGSANLEITUNG.md",
    "docs/INSTALL_FROM_ARCHIVE.md",
]

# Top-level directories to include wholesale (in addition to REQUIRED_FILES),
# so that any file added later without updating REQUIRED_FILES is still
# packaged. REQUIRED_FILES remains the source of truth for *verification*.
INCLUDE_DIRS = ["src", "tests", "docs", "man", "scripts"]
INCLUDE_FILES = [
    "pyproject.toml",
    "README.md",
    "LICENSE",
    "CHANGELOG.md",
    "requirements.txt",
    "requirements-dev.txt",
    "llama.cpp.config.example",
    "build_archive.py",
]

EXCLUDE_DIR_NAMES = {"__pycache__", ".pytest_cache", ".venv", "venv", ".git", "*.egg-info"}
EXCLUDE_SUFFIXES = {".pyc", ".pyo"}

ARCHIVE_ROOT_NAME = "llamacppctl"  # top-level directory name inside the tarball


class BuildError(RuntimeError):
    pass


def log(msg: str) -> None:
    print(f"[build_archive] {msg}")


def verify_required_files(project_dir: Path) -> None:
    log("Verifying required files are present...")
    missing = []
    for rel in REQUIRED_FILES:
        p = project_dir / rel
        if not p.is_file():
            missing.append(rel)
    if missing:
        raise BuildError(
            "Missing required files, refusing to build archive:\n  "
            + "\n  ".join(missing)
        )
    log(f"  All {len(REQUIRED_FILES)} required files present.")


def verify_dependencies_declared(project_dir: Path) -> list:
    """Parses pyproject.toml and returns the declared runtime dependency
    specifiers.

    pyproject.toml is the single source of truth for dependencies
    (requirements.txt merely installs the package), so there is no requirements
    mirror to cross-check. Raises BuildError if none are declared.
    """
    log("Reading declared runtime dependencies from pyproject.toml...")
    pyproject_path = project_dir / "pyproject.toml"

    try:
        import tomllib  # Python 3.11+
    except ModuleNotFoundError:
        try:
            import tomli as tomllib  # type: ignore
        except ModuleNotFoundError:
            tomllib = None

    if tomllib is not None:
        with open(pyproject_path, "rb") as f:
            data = tomllib.load(f)
        deps = data.get("project", {}).get("dependencies", [])
    else:
        # Fallback: minimal manual extraction, good enough for verification
        # purposes when neither tomllib nor tomli is available.
        log("  (tomllib/tomli unavailable, falling back to manual parse)")
        deps = []
        in_deps = False
        for line in pyproject_path.read_text(encoding="utf-8").splitlines():
            stripped = line.strip()
            if stripped.startswith("dependencies"):
                in_deps = True
                if "[" in stripped and "]" in stripped:
                    inner = stripped.split("[", 1)[1].rsplit("]", 1)[0]
                    deps = [d.strip().strip('"').strip("'") for d in inner.split(",") if d.strip()]
                    in_deps = False
                continue
            if in_deps:
                if "]" in stripped:
                    in_deps = False
                    stripped = stripped.split("]", 1)[0]
                cleaned = stripped.strip().rstrip(",").strip('"').strip("'")
                if cleaned:
                    deps.append(cleaned)

    if not deps:
        raise BuildError("No runtime dependencies found in pyproject.toml [project.dependencies]")

    log(f"  pyproject.toml dependencies: {deps}")
    return deps


def run_pip_dependency_check(project_dir: Path, deps: list) -> None:
    """Creates a throwaway virtualenv and does a real `pip install` of just
    the declared dependencies (not the project itself) to confirm they are
    resolvable from PyPI. This is a stronger guarantee than a --dry-run
    against an environment that might already happen to have them cached."""
    log("Verifying dependencies are installable (throwaway virtualenv)...")
    import tempfile

    with tempfile.TemporaryDirectory(prefix="llamacppctl-depcheck-") as tmp:
        venv_dir = Path(tmp) / "venv"
        venv.EnvBuilder(with_pip=True, clear=True).create(venv_dir)
        pip_bin = venv_dir / "bin" / "pip"
        if not pip_bin.exists():
            pip_bin = venv_dir / "Scripts" / "pip.exe"  # Windows fallback

        cmd = [str(pip_bin), "install", "--quiet", *deps]
        result = subprocess.run(cmd, capture_output=True, text=True)
        if result.returncode != 0:
            raise BuildError(
                "Dependency installability check failed:\n"
                f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
            )
    log("  All declared dependencies installed successfully in an isolated environment.")


def run_tests(project_dir: Path) -> None:
    log("Running test suite (pytest) before packaging...")
    result = subprocess.run(
        [sys.executable, "-m", "pytest", "-q"],
        cwd=str(project_dir),
        capture_output=True,
        text=True,
    )
    print(result.stdout)
    if result.returncode != 0:
        print(result.stderr, file=sys.stderr)
        raise BuildError(f"Test suite failed (exit code {result.returncode}); refusing to package.")
    log("  Test suite passed.")


def _tar_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None:
    parts = Path(tarinfo.name).parts
    for part in parts:
        if part in EXCLUDE_DIR_NAMES or part.endswith(".egg-info"):
            return None
    if Path(tarinfo.name).suffix in EXCLUDE_SUFFIXES:
        return None
    # Normalize metadata for reproducibility.
    tarinfo.uid = 0
    tarinfo.gid = 0
    tarinfo.uname = ""
    tarinfo.gname = ""
    return tarinfo


def build_tarball(project_dir: Path, output_path: Path) -> None:
    log(f"Building archive: {output_path}")
    output_path.parent.mkdir(parents=True, exist_ok=True)

    with tarfile.open(output_path, "w:gz") as tar:
        for rel in INCLUDE_FILES:
            src = project_dir / rel
            if src.is_file():
                tar.add(src, arcname=f"{ARCHIVE_ROOT_NAME}/{rel}", filter=_tar_filter)

        for dirname in INCLUDE_DIRS:
            src_dir = project_dir / dirname
            if src_dir.is_dir():
                tar.add(src_dir, arcname=f"{ARCHIVE_ROOT_NAME}/{dirname}", filter=_tar_filter)

    log(f"  Archive written: {output_path} ({output_path.stat().st_size:,} bytes)")


def verify_tarball(output_path: Path) -> None:
    log("Re-opening archive to verify contents...")
    with tarfile.open(output_path, "r:gz") as tar:
        names = set(tar.getnames())

    missing = []
    for rel in REQUIRED_FILES:
        expected = f"{ARCHIVE_ROOT_NAME}/{rel}"
        if expected not in names:
            missing.append(expected)

    if missing:
        raise BuildError(
            "Archive verification failed, required files missing from tarball:\n  "
            + "\n  ".join(missing)
        )

    log(f"  Verified {len(REQUIRED_FILES)} required files are present in the archive.")
    log(f"  Total archive members: {len(names)}")


def smoke_test_install(output_path: Path) -> None:
    """Extracts the tarball into a temp dir, installs it in a throwaway
    virtualenv, and confirms the `llamacppctl` console script works via
    --print-effective-config and --dry-run (no Docker daemon required for
    either)."""
    log("Smoke-testing installation from the built archive...")
    import tempfile

    with tempfile.TemporaryDirectory(prefix="llamacppctl-smoketest-") as tmp:
        tmp_path = Path(tmp)
        with tarfile.open(output_path, "r:gz") as tar:
            tar.extractall(tmp_path)

        extracted_project = tmp_path / ARCHIVE_ROOT_NAME
        venv_dir = tmp_path / "venv"
        venv.EnvBuilder(with_pip=True, clear=True).create(venv_dir)
        pip_bin = venv_dir / "bin" / "pip"
        llamacppctl_bin = venv_dir / "bin" / "llamacppctl"

        result = subprocess.run(
            [str(pip_bin), "install", "--quiet", str(extracted_project)],
            capture_output=True,
            text=True,
        )
        if result.returncode != 0:
            raise BuildError(
                "Smoke-test install failed:\n"
                f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
            )
        log("  Package installed successfully from the extracted archive.")

        config_example = extracted_project / "llama.cpp.config.example"

        result = subprocess.run(
            [
                str(llamacppctl_bin),
                "--print-effective-config",
                "--config", str(config_example),
                "--profile", "qwen35b",
                "--start",
                "--dry-run",
            ],
            capture_output=True,
            text=True,
        )
        print(result.stdout)

        docker_unavailable = "docker is not available on PATH" in (result.stdout + result.stderr)

        if result.returncode != 0 and not docker_unavailable:
            print(result.stderr, file=sys.stderr)
            raise BuildError(
                f"Smoke test invocation failed unexpectedly (exit code {result.returncode})."
            )

        if docker_unavailable:
            log(
                "  Console script and argument/config resolution work end-to-end; "
                "exited cleanly at the expected Docker-unavailable check "
                "(no Docker daemon in this environment -- this is not a packaging defect)."
            )
        else:
            log("  --print-effective-config / --dry-run smoke test succeeded.")


def main(argv: list | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--project-dir",
        default=str(Path(__file__).resolve().parent),
        help="Path to the llamacppctl project directory (default: this script's directory)",
    )
    parser.add_argument(
        "--output",
        default=None,
        help="Output path for the tar.gz (default: <project-dir>/../llamacppctl-installable.tar.gz)",
    )
    parser.add_argument("--skip-tests", action="store_true", help="Skip running pytest before packaging")
    parser.add_argument(
        "--skip-pip-check",
        action="store_true",
        help="Skip the throwaway-virtualenv dependency installability check",
    )
    parser.add_argument(
        "--skip-smoke-test",
        action="store_true",
        help="Skip extracting + installing the built archive as a final smoke test",
    )
    args = parser.parse_args(argv)

    project_dir = Path(args.project_dir).resolve()
    output_path = (
        Path(args.output).resolve()
        if args.output
        else project_dir.parent / "llamacppctl-installable.tar.gz"
    )

    try:
        verify_required_files(project_dir)
        deps = verify_dependencies_declared(project_dir)

        if not args.skip_pip_check:
            run_pip_dependency_check(project_dir, deps)
        else:
            log("Skipping pip dependency installability check (--skip-pip-check).")

        if not args.skip_tests:
            run_tests(project_dir)
        else:
            log("Skipping test suite (--skip-tests).")

        build_tarball(project_dir, output_path)
        verify_tarball(output_path)

        if not args.skip_smoke_test:
            smoke_test_install(output_path)
        else:
            log("Skipping install smoke test (--skip-smoke-test).")

    except BuildError as exc:
        print(f"\n[build_archive] BUILD FAILED: {exc}", file=sys.stderr)
        return 1

    log("")
    log("Build and verification complete.")
    log(f"Archive: {output_path}")
    log(f"Size:    {output_path.stat().st_size:,} bytes")
    log("")
    log("To install on the target machine:")
    log(f"  tar xzf {output_path.name}")
    log(f"  cd {ARCHIVE_ROOT_NAME}")
    log("  python3 -m venv .venv && source .venv/bin/activate")
    log("  pip install .")
    return 0


if __name__ == "__main__":
    sys.exit(main())