llamacppctl/build_archive.py

#!/usr/bin/env python3
"""build_archive.py

Assembles ``llamacppctl-installable.tar.gz`` directly on the local machine
from the llamacppctl project files, and verifies that all required files and
declared dependencies are present and consistent before/after packaging.

This script is intentionally self-contained (standard library only, plus an
optional import of ``tomllib``/``tomli`` purely for dependency verification)
so it can be run directly, without first installing the llamacppctl package
itself.

Usage
-----
    python3 build_archive.py
    python3 build_archive.py --project-dir /path/to/llamacppctl --output /path/to/llamacppctl-installable.tar.gz
    python3 build_archive.py --skip-tests          # skip running pytest before packaging
    python3 build_archive.py --skip-pip-check      # skip the pip dependency dry-run

Exit codes
----------
    0  archive built and verified successfully
    1  a required file was missing, a test failed, or verification failed
"""

from __future__ import annotations

import argparse
import subprocess
import sys
import tarfile
import venv
from pathlib import Path

# --------------------------------------------------------------------------
# Required file manifest.
#
# Every path here is relative to the project directory and MUST exist before
# packaging. This is the "verifies that all dependencies are correctly
# included" contract: the archive is only built if every one of these is
# present, and re-opened afterwards to confirm every one of them made it
# into the tarball unmodified.
# --------------------------------------------------------------------------
REQUIRED_FILES = [
    "pyproject.toml",
    "README.md",
    "requirements.txt",
    "requirements-dev.txt",
    "llama.cpp.config.example",
    "docs/SECURITY_AND_OPERATIONS.md",
    "man/llamacppctl.1",
    "src/llamacppctl/__init__.py",
    "src/llamacppctl/py.typed",
    "src/llamacppctl/cli.py",
    "src/llamacppctl/schema.py",
    "src/llamacppctl/config.py",
    "src/llamacppctl/prompt_io.py",
    "src/llamacppctl/docker_ops.py",
    "src/llamacppctl/http_ops.py",
    "src/llamacppctl/lock_ops.py",
    "src/llamacppctl/actions.py",
    "src/llamacppctl/main.py",
    "tests/conftest.py",
    "tests/test_prompt_sources.py",
    "tests/test_prompt_files.py",
    "tests/test_prompt_urls.py",
    "tests/test_config.py",
    "tests/test_cli.py",
    "tests/test_docker_ops.py",
    "tests/test_http_ops.py",
    "tests/test_lock_ops.py",
    "tests/test_actions.py",
    "docs/How_to_use.md",
    "scripts/smoke.sh",
]

# Top-level directories to include wholesale (in addition to REQUIRED_FILES),
# so that any file added later without updating REQUIRED_FILES is still
# packaged. REQUIRED_FILES remains the source of truth for *verification*.
INCLUDE_DIRS = ["src", "tests", "docs", "man", "scripts"]
INCLUDE_FILES = [
    "pyproject.toml",
    "README.md",
    "requirements.txt",
    "requirements-dev.txt",
    "llama.cpp.config.example",
    "build_archive.py",
]

EXCLUDE_DIR_NAMES = {"__pycache__", ".pytest_cache", ".venv", "venv", ".git", "*.egg-info"}
EXCLUDE_SUFFIXES = {".pyc", ".pyo"}

ARCHIVE_ROOT_NAME = "llamacppctl"  # top-level directory name inside the tarball


class BuildError(RuntimeError):
    pass


def log(msg: str) -> None:
    print(f"[build_archive] {msg}")


def verify_required_files(project_dir: Path) -> None:
    log("Verifying required files are present...")
    missing = []
    for rel in REQUIRED_FILES:
        p = project_dir / rel
        if not p.is_file():
            missing.append(rel)
    if missing:
        raise BuildError(
            "Missing required files, refusing to build archive:\n  "
            + "\n  ".join(missing)
        )
    log(f"  All {len(REQUIRED_FILES)} required files present.")


def verify_dependencies_declared(project_dir: Path) -> list:
    """Parses pyproject.toml and cross-checks it against requirements.txt.

    Returns the list of runtime dependency specifiers declared in
    pyproject.toml. Raises BuildError on any mismatch.
    """
    log("Verifying declared dependencies are consistent...")
    pyproject_path = project_dir / "pyproject.toml"
    requirements_path = project_dir / "requirements.txt"

    try:
        import tomllib  # Python 3.11+
    except ModuleNotFoundError:
        try:
            import tomli as tomllib  # type: ignore
        except ModuleNotFoundError:
            tomllib = None

    if tomllib is not None:
        with open(pyproject_path, "rb") as f:
            data = tomllib.load(f)
        deps = data.get("project", {}).get("dependencies", [])
    else:
        # Fallback: minimal manual extraction, good enough for verification
        # purposes when neither tomllib nor tomli is available.
        log("  (tomllib/tomli unavailable, falling back to manual parse)")
        deps = []
        in_deps = False
        for line in pyproject_path.read_text(encoding="utf-8").splitlines():
            stripped = line.strip()
            if stripped.startswith("dependencies"):
                in_deps = True
                if "[" in stripped and "]" in stripped:
                    inner = stripped.split("[", 1)[1].rsplit("]", 1)[0]
                    deps = [d.strip().strip('"').strip("'") for d in inner.split(",") if d.strip()]
                    in_deps = False
                continue
            if in_deps:
                if "]" in stripped:
                    in_deps = False
                    stripped = stripped.split("]", 1)[0]
                cleaned = stripped.strip().rstrip(",").strip('"').strip("'")
                if cleaned:
                    deps.append(cleaned)

    if not deps:
        raise BuildError("No runtime dependencies found in pyproject.toml [project.dependencies]")

    req_text = requirements_path.read_text(encoding="utf-8")
    req_names = set()
    for line in req_text.splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        # crude package-name extraction, e.g. "requests>=2.31,<3" -> "requests"
        name = line
        for sep in (">=", "<=", "==", "!=", ">", "<", "~="):
            if sep in name:
                name = name.split(sep, 1)[0]
        req_names.add(name.strip().lower())

    missing_from_requirements = []
    for dep in deps:
        name = dep
        for sep in (">=", "<=", "==", "!=", ">", "<", "~="):
            if sep in name:
                name = name.split(sep, 1)[0]
        name = name.strip().lower()
        if name not in req_names:
            missing_from_requirements.append(dep)

    if missing_from_requirements:
        raise BuildError(
            "pyproject.toml declares dependencies not mirrored in requirements.txt:\n  "
            + "\n  ".join(missing_from_requirements)
        )

    log(f"  pyproject.toml dependencies: {deps}")
    log("  requirements.txt is consistent with pyproject.toml.")
    return deps


def run_pip_dependency_check(project_dir: Path, deps: list) -> None:
    """Creates a throwaway virtualenv and does a real `pip install` of just
    the declared dependencies (not the project itself) to confirm they are
    resolvable from PyPI. This is a stronger guarantee than a --dry-run
    against an environment that might already happen to have them cached."""
    log("Verifying dependencies are installable (throwaway virtualenv)...")
    import tempfile

    with tempfile.TemporaryDirectory(prefix="llamacppctl-depcheck-") as tmp:
        venv_dir = Path(tmp) / "venv"
        venv.EnvBuilder(with_pip=True, clear=True).create(venv_dir)
        pip_bin = venv_dir / "bin" / "pip"
        if not pip_bin.exists():
            pip_bin = venv_dir / "Scripts" / "pip.exe"  # Windows fallback

        cmd = [str(pip_bin), "install", "--quiet", *deps]
        result = subprocess.run(cmd, capture_output=True, text=True)
        if result.returncode != 0:
            raise BuildError(
                "Dependency installability check failed:\n"
                f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
            )
    log("  All declared dependencies installed successfully in an isolated environment.")


def run_tests(project_dir: Path) -> None:
    log("Running test suite (pytest) before packaging...")
    result = subprocess.run(
        [sys.executable, "-m", "pytest", "-q"],
        cwd=str(project_dir),
        capture_output=True,
        text=True,
    )
    print(result.stdout)
    if result.returncode != 0:
        print(result.stderr, file=sys.stderr)
        raise BuildError(f"Test suite failed (exit code {result.returncode}); refusing to package.")
    log("  Test suite passed.")


def _tar_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None:
    parts = Path(tarinfo.name).parts
    for part in parts:
        if part in EXCLUDE_DIR_NAMES or part.endswith(".egg-info"):
            return None
    if Path(tarinfo.name).suffix in EXCLUDE_SUFFIXES:
        return None
    # Normalize metadata for reproducibility.
    tarinfo.uid = 0
    tarinfo.gid = 0
    tarinfo.uname = ""
    tarinfo.gname = ""
    return tarinfo


def build_tarball(project_dir: Path, output_path: Path) -> None:
    log(f"Building archive: {output_path}")
    output_path.parent.mkdir(parents=True, exist_ok=True)

    with tarfile.open(output_path, "w:gz") as tar:
        for rel in INCLUDE_FILES:
            src = project_dir / rel
            if src.is_file():
                tar.add(src, arcname=f"{ARCHIVE_ROOT_NAME}/{rel}", filter=_tar_filter)

        for dirname in INCLUDE_DIRS:
            src_dir = project_dir / dirname
            if src_dir.is_dir():
                tar.add(src_dir, arcname=f"{ARCHIVE_ROOT_NAME}/{dirname}", filter=_tar_filter)

    log(f"  Archive written: {output_path} ({output_path.stat().st_size:,} bytes)")


def verify_tarball(output_path: Path) -> None:
    log("Re-opening archive to verify contents...")
    with tarfile.open(output_path, "r:gz") as tar:
        names = set(tar.getnames())

    missing = []
    for rel in REQUIRED_FILES:
        expected = f"{ARCHIVE_ROOT_NAME}/{rel}"
        if expected not in names:
            missing.append(expected)

    if missing:
        raise BuildError(
            "Archive verification failed, required files missing from tarball:\n  "
            + "\n  ".join(missing)
        )

    log(f"  Verified {len(REQUIRED_FILES)} required files are present in the archive.")
    log(f"  Total archive members: {len(names)}")


def smoke_test_install(output_path: Path) -> None:
    """Extracts the tarball into a temp dir, installs it in a throwaway
    virtualenv, and confirms the `llamacppctl` console script works via
    --print-effective-config and --dry-run (no Docker daemon required for
    either)."""
    log("Smoke-testing installation from the built archive...")
    import tempfile

    with tempfile.TemporaryDirectory(prefix="llamacppctl-smoketest-") as tmp:
        tmp_path = Path(tmp)
        with tarfile.open(output_path, "r:gz") as tar:
            tar.extractall(tmp_path)

        extracted_project = tmp_path / ARCHIVE_ROOT_NAME
        venv_dir = tmp_path / "venv"
        venv.EnvBuilder(with_pip=True, clear=True).create(venv_dir)
        pip_bin = venv_dir / "bin" / "pip"
        py_bin = venv_dir / "bin" / "python"
        llamacppctl_bin = venv_dir / "bin" / "llamacppctl"

        result = subprocess.run(
            [str(pip_bin), "install", "--quiet", str(extracted_project)],
            capture_output=True,
            text=True,
        )
        if result.returncode != 0:
            raise BuildError(
                "Smoke-test install failed:\n"
                f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
            )
        log("  Package installed successfully from the extracted archive.")

        config_example = extracted_project / "llama.cpp.config.example"

        result = subprocess.run(
            [
                str(llamacppctl_bin),
                "--print-effective-config",
                "--config", str(config_example),
                "--profile", "qwen35",
                "--start",
                "--dry-run",
            ],
            capture_output=True,
            text=True,
        )
        print(result.stdout)

        docker_unavailable = "docker is not available on PATH" in (result.stdout + result.stderr)

        if result.returncode != 0 and not docker_unavailable:
            print(result.stderr, file=sys.stderr)
            raise BuildError(
                f"Smoke test invocation failed unexpectedly (exit code {result.returncode})."
            )

        if docker_unavailable:
            log(
                "  Console script and argument/config resolution work end-to-end; "
                "exited cleanly at the expected Docker-unavailable check "
                "(no Docker daemon in this environment -- this is not a packaging defect)."
            )
        else:
            log("  --print-effective-config / --dry-run smoke test succeeded.")


def main(argv: list | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--project-dir",
        default=str(Path(__file__).resolve().parent),
        help="Path to the llamacppctl project directory (default: this script's directory)",
    )
    parser.add_argument(
        "--output",
        default=None,
        help="Output path for the tar.gz (default: <project-dir>/../llamacppctl-installable.tar.gz)",
    )
    parser.add_argument("--skip-tests", action="store_true", help="Skip running pytest before packaging")
    parser.add_argument(
        "--skip-pip-check",
        action="store_true",
        help="Skip the throwaway-virtualenv dependency installability check",
    )
    parser.add_argument(
        "--skip-smoke-test",
        action="store_true",
        help="Skip extracting + installing the built archive as a final smoke test",
    )
    args = parser.parse_args(argv)

    project_dir = Path(args.project_dir).resolve()
    output_path = (
        Path(args.output).resolve()
        if args.output
        else project_dir.parent / "llamacppctl-installable.tar.gz"
    )

    try:
        verify_required_files(project_dir)
        deps = verify_dependencies_declared(project_dir)

        if not args.skip_pip_check:
            run_pip_dependency_check(project_dir, deps)
        else:
            log("Skipping pip dependency installability check (--skip-pip-check).")

        if not args.skip_tests:
            run_tests(project_dir)
        else:
            log("Skipping test suite (--skip-tests).")

        build_tarball(project_dir, output_path)
        verify_tarball(output_path)

        if not args.skip_smoke_test:
            smoke_test_install(output_path)
        else:
            log("Skipping install smoke test (--skip-smoke-test).")

    except BuildError as exc:
        print(f"\n[build_archive] BUILD FAILED: {exc}", file=sys.stderr)
        return 1

    log("")
    log("Build and verification complete.")
    log(f"Archive: {output_path}")
    log(f"Size:    {output_path.stat().st_size:,} bytes")
    log("")
    log("To install on the target machine:")
    log(f"  tar xzf {output_path.name}")
    log(f"  cd {ARCHIVE_ROOT_NAME}")
    log("  python3 -m venv .venv && source .venv/bin/activate")
    log("  pip install .")
    return 0


if __name__ == "__main__":
    sys.exit(main())