diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fd3f4e..1463e60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Per-profile chat streaming and chat-request timeouts: `stream`, `read_timeout` + and `connect_timeout` are now resolvable in `[default]`/`[model.]` + (CLI `--stream`/`--read-timeout`/`--connect-timeout` still override). + +### Changed +- The chat request now honors the configured `(connect, read)` timeout instead + of a hard-coded 30 s; the default `read_timeout` is 600 s so slow reasoning + models are no longer cut off mid-generation. + ## [0.1.0] - 2026-07-07 Initial release. Replaces the previous collection of shell scripts diff --git a/llama.cpp.config.example b/llama.cpp.config.example index 628da02..b411535 100644 --- a/llama.cpp.config.example +++ b/llama.cpp.config.example @@ -62,6 +62,14 @@ api_key = max_tokens = 2048 # chat_temperature leer lassen -> die Server-Temperatur (temp) gilt. chat_temperature = +# Chat-Antwort standardmäßig token-weise streamen (wie --stream). Für langsame +# Reasoning-Modelle angenehm, weil sofort Ausgabe erscheint. +stream = false +# (connect, read)-Timeout des Chat-Requests in Sekunden. read_timeout muss die +# volle Generierungszeit abdecken; Reasoning-Modelle brauchen viel -> hoch +# setzen. --read-timeout/--connect-timeout auf der CLI überschreiben das. +read_timeout = 600 +connect_timeout = 10 # Example second model profile, pinned to the second GPU (e.g. RTX 3090 #2) # with its own port and container name so it can run alongside [default]. diff --git a/src/llamacppctl/actions.py b/src/llamacppctl/actions.py index f8cd8ad..9011338 100644 --- a/src/llamacppctl/actions.py +++ b/src/llamacppctl/actions.py @@ -233,7 +233,8 @@ def do_chat(cfg: ServerConfig, prompt_cfg: PromptConfig, args) -> int: print("--chat requires a user prompt", file=sys.stderr) return EXIT_GENERAL - if getattr(args, "stream", False): + # Streaming can be requested via --stream OR enabled per profile in config. + if getattr(args, "stream", False) or getattr(prompt_cfg, "stream", False): return _do_chat_stream(cfg, prompt_cfg, args) try: diff --git a/src/llamacppctl/cli.py b/src/llamacppctl/cli.py index 2a99f98..11ed47f 100644 --- a/src/llamacppctl/cli.py +++ b/src/llamacppctl/cli.py @@ -103,8 +103,11 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--no-allow-symlinks", dest="allow_symlinks", action="store_false") p.add_argument("--max-input-bytes", type=int, default=1_048_576) p.add_argument("--url-allow-host", action="append", default=[]) - p.add_argument("--connect-timeout", type=float, default=3.0) - p.add_argument("--read-timeout", type=float, default=10.0) + # Default None so a config-provided value can win; a fallback is applied at + # the point of use (URL fetch: 3s/10s; chat request: profile read_timeout). + # When set, these also override the chat request's (connect, read) timeout. + p.add_argument("--connect-timeout", type=float, default=None) + p.add_argument("--read-timeout", type=float, default=None) # --- Chat request parameters (for --chat and the --start reply) --- p.add_argument( @@ -166,7 +169,9 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> if args.max_tokens is not None and args.max_tokens <= 0: parser.error("--max-tokens must be > 0") - if args.connect_timeout <= 0 or args.read_timeout <= 0: + if (args.connect_timeout is not None and args.connect_timeout <= 0) or ( + args.read_timeout is not None and args.read_timeout <= 0 + ): parser.error("timeouts must be > 0") if args.allow_private_url and args.url_allow_host: diff --git a/src/llamacppctl/config.py b/src/llamacppctl/config.py index 301ba9a..684a6e9 100644 --- a/src/llamacppctl/config.py +++ b/src/llamacppctl/config.py @@ -248,11 +248,29 @@ def resolve_effective_config(args) -> tuple: if getattr(args, "chat_temp", None) is not None: chat_temp = args.chat_temp + # Streaming: enabled per profile via `stream = true`, or with --stream. + stream = str(merged.get("stream", "")).strip().lower() in TRUE_STRINGS + if getattr(args, "stream", False): + stream = True + + # Chat request timeouts: config value first, CLI flag overrides. These are + # the (connect, read) timeouts of the chat POST; read_timeout must cover the + # model's full generation time (raise it for slow reasoning models). + read_timeout = float(merged.get("read_timeout") or 600.0) + if getattr(args, "read_timeout", None) is not None: + read_timeout = args.read_timeout + connect_timeout = float(merged.get("connect_timeout") or 10.0) + if getattr(args, "connect_timeout", None) is not None: + connect_timeout = args.connect_timeout + prompt_cfg = PromptConfig( system_prompt=system_prompt, user_prompt=getattr(args, "_resolved_user_prompt", None), max_tokens=max_tokens, temperature=chat_temp, + stream=stream, + connect_timeout=connect_timeout, + read_timeout=read_timeout, ) return server_cfg, prompt_cfg diff --git a/src/llamacppctl/http_ops.py b/src/llamacppctl/http_ops.py index f414914..420c0fa 100644 --- a/src/llamacppctl/http_ops.py +++ b/src/llamacppctl/http_ops.py @@ -88,18 +88,21 @@ def chat_completion(cfg: ServerConfig, prompt_cfg: PromptConfig) -> requests.Res def chat_completion_text( - cfg: ServerConfig, prompt_cfg: PromptConfig, timeout: float = 30.0 + cfg: ServerConfig, prompt_cfg: PromptConfig, timeout: Optional[tuple] = None ) -> ChatReply: """High-level chat helper for --chat and the --start/--check post-checks. Returns a ChatReply (content + finish_reason), raising HttpError on - transport, HTTP status, JSON decoding, or malformed-response failures.""" + transport, HTTP status, JSON decoding, or malformed-response failures. + + The (connect, read) timeout defaults to the prompt profile's configured + values so slow reasoning models are not cut off mid-generation.""" try: resp = requests.post( chat_url(cfg), json=_chat_payload(cfg, prompt_cfg), headers=_auth_headers(cfg), - timeout=timeout, + timeout=timeout or (prompt_cfg.connect_timeout, prompt_cfg.read_timeout), ) except requests.RequestException as exc: raise HttpError(f"chat completion request failed: {exc}") from exc @@ -137,7 +140,7 @@ def stream_chat( json=payload, headers=_auth_headers(cfg), stream=True, - timeout=timeout or (10.0, 600.0), + timeout=timeout or (prompt_cfg.connect_timeout, prompt_cfg.read_timeout), ) except requests.RequestException as exc: raise HttpError(f"chat completion request failed: {exc}") from exc diff --git a/src/llamacppctl/main.py b/src/llamacppctl/main.py index 3bf16c1..b633b96 100644 --- a/src/llamacppctl/main.py +++ b/src/llamacppctl/main.py @@ -24,8 +24,8 @@ from .prompt_io import InputPolicy, PromptSourceError, load_prompt_source, resol def _build_input_policy(args) -> InputPolicy: return InputPolicy( max_input_bytes=args.max_input_bytes, - connect_timeout=args.connect_timeout, - read_timeout=args.read_timeout, + connect_timeout=args.connect_timeout if args.connect_timeout is not None else 3.0, + read_timeout=args.read_timeout if args.read_timeout is not None else 10.0, allow_insecure_http=args.allow_insecure_http, allow_private_url=args.allow_private_url, allow_ip_host=args.allow_ip_host, diff --git a/src/llamacppctl/schema.py b/src/llamacppctl/schema.py index ab057d7..13018c0 100644 --- a/src/llamacppctl/schema.py +++ b/src/llamacppctl/schema.py @@ -73,6 +73,12 @@ class PromptConfig: # None => omit from the request so the server's configured --temp applies. temperature: Optional[float] = None stream: bool = False + # HTTP timeouts for the chat request itself (seconds). read_timeout must + # cover the model's full generation time: reasoning models can "think" for + # a long time before the first/last token, so the default is generous. Both + # are configurable per profile (read_timeout/connect_timeout in the config). + connect_timeout: float = 10.0 + read_timeout: float = 600.0 @dataclass diff --git a/tests/test_config.py b/tests/test_config.py index 1b162dc..38ddb2b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -258,3 +258,33 @@ def test_chat_params_cli_overrides_config(tmp_path): _, prompt_cfg = resolve_effective_config(args) assert prompt_cfg.max_tokens == 512 assert prompt_cfg.temperature == 0.1 + + +def test_stream_and_timeouts_default(tmp_path): + cfg_path = _write_config(tmp_path) + _, prompt_cfg = resolve_effective_config(_parse(["--start", "--config", str(cfg_path)])) + assert prompt_cfg.stream is False + assert prompt_cfg.read_timeout == 600.0 # generous default for reasoning models + assert prompt_cfg.connect_timeout == 10.0 + + +def test_stream_and_read_timeout_from_config(tmp_path): + cfg = CONFIG_BASIC.replace( + "poll_interval = 2", + "poll_interval = 2\nstream = true\nread_timeout = 900\nconnect_timeout = 5", + ) + cfg_path = _write_config(tmp_path, cfg) + _, prompt_cfg = resolve_effective_config(_parse(["--start", "--config", str(cfg_path)])) + assert prompt_cfg.stream is True + assert prompt_cfg.read_timeout == 900.0 + assert prompt_cfg.connect_timeout == 5.0 + + +def test_stream_and_read_timeout_cli_overrides_config(tmp_path): + cfg = CONFIG_BASIC.replace("poll_interval = 2", "poll_interval = 2\nread_timeout = 900") + cfg_path = _write_config(tmp_path, cfg) + # --stream forces streaming on; --read-timeout overrides the config value. + args = _parse(["--chat", "-p", "hi", "--config", str(cfg_path), "--stream", "--read-timeout", "42"]) + _, prompt_cfg = resolve_effective_config(args) + assert prompt_cfg.stream is True + assert prompt_cfg.read_timeout == 42.0