From 6076e3bcb828270b30f17c41fe9b651389d2b52f Mon Sep 17 00:00:00 2001 From: dschlueter Date: Thu, 9 Jul 2026 18:26:33 +0200 Subject: [PATCH 1/2] feat(chat): make streaming and chat timeouts per-profile configurable The chat request previously used a hard-coded 30 s timeout and ignored --read-timeout entirely (that flag only bounded URL prompt fetching), so slow reasoning models were cut off mid-generation unless one remembered to pass --stream (which used a separate hard-coded 600 s). Resolve `stream`, `read_timeout` and `connect_timeout` from [default]/[model.] into PromptConfig and wire the (connect, read) timeout into both the streaming and non-streaming chat calls. CLI --stream/--read-timeout/--connect-timeout still override; the two timeout flags default to None so a config value can win, with the URL-fetch fallbacks (3 s/10 s) preserved. Default chat read_timeout is now 600 s. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 10 ++++++++++ llama.cpp.config.example | 8 ++++++++ src/llamacppctl/actions.py | 3 ++- src/llamacppctl/cli.py | 11 ++++++++--- src/llamacppctl/config.py | 18 ++++++++++++++++++ src/llamacppctl/http_ops.py | 11 +++++++---- src/llamacppctl/main.py | 4 ++-- src/llamacppctl/schema.py | 6 ++++++ tests/test_config.py | 30 ++++++++++++++++++++++++++++++ 9 files changed, 91 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fd3f4e..1463e60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Per-profile chat streaming and chat-request timeouts: `stream`, `read_timeout` + and `connect_timeout` are now resolvable in `[default]`/`[model.]` + (CLI `--stream`/`--read-timeout`/`--connect-timeout` still override). + +### Changed +- The chat request now honors the configured `(connect, read)` timeout instead + of a hard-coded 30 s; the default `read_timeout` is 600 s so slow reasoning + models are no longer cut off mid-generation. + ## [0.1.0] - 2026-07-07 Initial release. Replaces the previous collection of shell scripts diff --git a/llama.cpp.config.example b/llama.cpp.config.example index 628da02..b411535 100644 --- a/llama.cpp.config.example +++ b/llama.cpp.config.example @@ -62,6 +62,14 @@ api_key = max_tokens = 2048 # chat_temperature leer lassen -> die Server-Temperatur (temp) gilt. chat_temperature = +# Chat-Antwort standardmäßig token-weise streamen (wie --stream). Für langsame +# Reasoning-Modelle angenehm, weil sofort Ausgabe erscheint. +stream = false +# (connect, read)-Timeout des Chat-Requests in Sekunden. read_timeout muss die +# volle Generierungszeit abdecken; Reasoning-Modelle brauchen viel -> hoch +# setzen. --read-timeout/--connect-timeout auf der CLI überschreiben das. +read_timeout = 600 +connect_timeout = 10 # Example second model profile, pinned to the second GPU (e.g. RTX 3090 #2) # with its own port and container name so it can run alongside [default]. diff --git a/src/llamacppctl/actions.py b/src/llamacppctl/actions.py index f8cd8ad..9011338 100644 --- a/src/llamacppctl/actions.py +++ b/src/llamacppctl/actions.py @@ -233,7 +233,8 @@ def do_chat(cfg: ServerConfig, prompt_cfg: PromptConfig, args) -> int: print("--chat requires a user prompt", file=sys.stderr) return EXIT_GENERAL - if getattr(args, "stream", False): + # Streaming can be requested via --stream OR enabled per profile in config. + if getattr(args, "stream", False) or getattr(prompt_cfg, "stream", False): return _do_chat_stream(cfg, prompt_cfg, args) try: diff --git a/src/llamacppctl/cli.py b/src/llamacppctl/cli.py index 2a99f98..11ed47f 100644 --- a/src/llamacppctl/cli.py +++ b/src/llamacppctl/cli.py @@ -103,8 +103,11 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--no-allow-symlinks", dest="allow_symlinks", action="store_false") p.add_argument("--max-input-bytes", type=int, default=1_048_576) p.add_argument("--url-allow-host", action="append", default=[]) - p.add_argument("--connect-timeout", type=float, default=3.0) - p.add_argument("--read-timeout", type=float, default=10.0) + # Default None so a config-provided value can win; a fallback is applied at + # the point of use (URL fetch: 3s/10s; chat request: profile read_timeout). + # When set, these also override the chat request's (connect, read) timeout. + p.add_argument("--connect-timeout", type=float, default=None) + p.add_argument("--read-timeout", type=float, default=None) # --- Chat request parameters (for --chat and the --start reply) --- p.add_argument( @@ -166,7 +169,9 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> if args.max_tokens is not None and args.max_tokens <= 0: parser.error("--max-tokens must be > 0") - if args.connect_timeout <= 0 or args.read_timeout <= 0: + if (args.connect_timeout is not None and args.connect_timeout <= 0) or ( + args.read_timeout is not None and args.read_timeout <= 0 + ): parser.error("timeouts must be > 0") if args.allow_private_url and args.url_allow_host: diff --git a/src/llamacppctl/config.py b/src/llamacppctl/config.py index 301ba9a..684a6e9 100644 --- a/src/llamacppctl/config.py +++ b/src/llamacppctl/config.py @@ -248,11 +248,29 @@ def resolve_effective_config(args) -> tuple: if getattr(args, "chat_temp", None) is not None: chat_temp = args.chat_temp + # Streaming: enabled per profile via `stream = true`, or with --stream. + stream = str(merged.get("stream", "")).strip().lower() in TRUE_STRINGS + if getattr(args, "stream", False): + stream = True + + # Chat request timeouts: config value first, CLI flag overrides. These are + # the (connect, read) timeouts of the chat POST; read_timeout must cover the + # model's full generation time (raise it for slow reasoning models). + read_timeout = float(merged.get("read_timeout") or 600.0) + if getattr(args, "read_timeout", None) is not None: + read_timeout = args.read_timeout + connect_timeout = float(merged.get("connect_timeout") or 10.0) + if getattr(args, "connect_timeout", None) is not None: + connect_timeout = args.connect_timeout + prompt_cfg = PromptConfig( system_prompt=system_prompt, user_prompt=getattr(args, "_resolved_user_prompt", None), max_tokens=max_tokens, temperature=chat_temp, + stream=stream, + connect_timeout=connect_timeout, + read_timeout=read_timeout, ) return server_cfg, prompt_cfg diff --git a/src/llamacppctl/http_ops.py b/src/llamacppctl/http_ops.py index f414914..420c0fa 100644 --- a/src/llamacppctl/http_ops.py +++ b/src/llamacppctl/http_ops.py @@ -88,18 +88,21 @@ def chat_completion(cfg: ServerConfig, prompt_cfg: PromptConfig) -> requests.Res def chat_completion_text( - cfg: ServerConfig, prompt_cfg: PromptConfig, timeout: float = 30.0 + cfg: ServerConfig, prompt_cfg: PromptConfig, timeout: Optional[tuple] = None ) -> ChatReply: """High-level chat helper for --chat and the --start/--check post-checks. Returns a ChatReply (content + finish_reason), raising HttpError on - transport, HTTP status, JSON decoding, or malformed-response failures.""" + transport, HTTP status, JSON decoding, or malformed-response failures. + + The (connect, read) timeout defaults to the prompt profile's configured + values so slow reasoning models are not cut off mid-generation.""" try: resp = requests.post( chat_url(cfg), json=_chat_payload(cfg, prompt_cfg), headers=_auth_headers(cfg), - timeout=timeout, + timeout=timeout or (prompt_cfg.connect_timeout, prompt_cfg.read_timeout), ) except requests.RequestException as exc: raise HttpError(f"chat completion request failed: {exc}") from exc @@ -137,7 +140,7 @@ def stream_chat( json=payload, headers=_auth_headers(cfg), stream=True, - timeout=timeout or (10.0, 600.0), + timeout=timeout or (prompt_cfg.connect_timeout, prompt_cfg.read_timeout), ) except requests.RequestException as exc: raise HttpError(f"chat completion request failed: {exc}") from exc diff --git a/src/llamacppctl/main.py b/src/llamacppctl/main.py index 3bf16c1..b633b96 100644 --- a/src/llamacppctl/main.py +++ b/src/llamacppctl/main.py @@ -24,8 +24,8 @@ from .prompt_io import InputPolicy, PromptSourceError, load_prompt_source, resol def _build_input_policy(args) -> InputPolicy: return InputPolicy( max_input_bytes=args.max_input_bytes, - connect_timeout=args.connect_timeout, - read_timeout=args.read_timeout, + connect_timeout=args.connect_timeout if args.connect_timeout is not None else 3.0, + read_timeout=args.read_timeout if args.read_timeout is not None else 10.0, allow_insecure_http=args.allow_insecure_http, allow_private_url=args.allow_private_url, allow_ip_host=args.allow_ip_host, diff --git a/src/llamacppctl/schema.py b/src/llamacppctl/schema.py index ab057d7..13018c0 100644 --- a/src/llamacppctl/schema.py +++ b/src/llamacppctl/schema.py @@ -73,6 +73,12 @@ class PromptConfig: # None => omit from the request so the server's configured --temp applies. temperature: Optional[float] = None stream: bool = False + # HTTP timeouts for the chat request itself (seconds). read_timeout must + # cover the model's full generation time: reasoning models can "think" for + # a long time before the first/last token, so the default is generous. Both + # are configurable per profile (read_timeout/connect_timeout in the config). + connect_timeout: float = 10.0 + read_timeout: float = 600.0 @dataclass diff --git a/tests/test_config.py b/tests/test_config.py index 1b162dc..38ddb2b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -258,3 +258,33 @@ def test_chat_params_cli_overrides_config(tmp_path): _, prompt_cfg = resolve_effective_config(args) assert prompt_cfg.max_tokens == 512 assert prompt_cfg.temperature == 0.1 + + +def test_stream_and_timeouts_default(tmp_path): + cfg_path = _write_config(tmp_path) + _, prompt_cfg = resolve_effective_config(_parse(["--start", "--config", str(cfg_path)])) + assert prompt_cfg.stream is False + assert prompt_cfg.read_timeout == 600.0 # generous default for reasoning models + assert prompt_cfg.connect_timeout == 10.0 + + +def test_stream_and_read_timeout_from_config(tmp_path): + cfg = CONFIG_BASIC.replace( + "poll_interval = 2", + "poll_interval = 2\nstream = true\nread_timeout = 900\nconnect_timeout = 5", + ) + cfg_path = _write_config(tmp_path, cfg) + _, prompt_cfg = resolve_effective_config(_parse(["--start", "--config", str(cfg_path)])) + assert prompt_cfg.stream is True + assert prompt_cfg.read_timeout == 900.0 + assert prompt_cfg.connect_timeout == 5.0 + + +def test_stream_and_read_timeout_cli_overrides_config(tmp_path): + cfg = CONFIG_BASIC.replace("poll_interval = 2", "poll_interval = 2\nread_timeout = 900") + cfg_path = _write_config(tmp_path, cfg) + # --stream forces streaming on; --read-timeout overrides the config value. + args = _parse(["--chat", "-p", "hi", "--config", str(cfg_path), "--stream", "--read-timeout", "42"]) + _, prompt_cfg = resolve_effective_config(args) + assert prompt_cfg.stream is True + assert prompt_cfg.read_timeout == 42.0 From 489af21fd2615777461ec2556fa9360fc0e1a070 Mon Sep 17 00:00:00 2001 From: dschlueter Date: Thu, 9 Jul 2026 18:26:40 +0200 Subject: [PATCH 2/2] docs(prompts): add RNG-uniform coding prompt and ornith35b usage Add coding_05_rng_uniform.md (cryptographically strong uniform float in [0.0, 1.0), 53-bit mantissa, bias-free, with distribution tests) and document running the coding prompts against the ornith35b profile, which ships coding-tuned sampling plus config-backed stream + read_timeout. Co-Authored-By: Claude Opus 4.8 --- example_user_prompts/README.md | 23 +++++++++++++++---- example_user_prompts/coding_05_rng_uniform.md | 10 ++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 example_user_prompts/coding_05_rng_uniform.md diff --git a/example_user_prompts/README.md b/example_user_prompts/README.md index 3287487..8bee342 100644 --- a/example_user_prompts/README.md +++ b/example_user_prompts/README.md @@ -13,7 +13,7 @@ Referenzmodell (zunächst): das Default-Modell aus `llama.cpp.config` |---|---| | `system_prompt_prosa.md` | `prosa_01_werkstatt.md`, `prosa_02_dystopie.md`, `prosa_03_essay.md`, `prosa_04_dialog.md` | | `system_prompt_reden.md` | `reden_01_abifeier.md`, `reden_02_windkraft.md`, `reden_03_umstrukturierung.md`, `reden_04_gedenken.md` | -| `system_prompt_coding.md` | `coding_01_jsonl_validator.md`, `coding_02_refactor.md`, `coding_03_ts_retry.md`, `coding_04_performance.md` | +| `system_prompt_coding.md` | `coding_01_jsonl_validator.md`, `coding_02_refactor.md`, `coding_03_ts_retry.md`, `coding_04_performance.md`, `coding_05_rng_uniform.md` | Jeder User-Prompt fordert gezielt Eigenschaften heraus, die der jeweilige System-Prompt verspricht (z. B. „show, don't tell" und Schlussbild bei Prosa; @@ -37,7 +37,20 @@ Hinweise: (`--max-tokens`) muss **Denk- plus Ausgabe-Tokens** abdecken — für Prosa/Reden großzügig wählen (z. B. 6000–10000), sonst bricht die sichtbare Antwort ab oder bleibt leer. -- **`--stream` benutzen.** Ohne Streaming wartet das Tool die komplette Antwort - in einem einzigen Read ab und läuft beim langen Reasoning-Output in den - Default-`--read-timeout` (30 s). Beim Streaming setzt jeder Token den Timeout - zurück. Für lange nicht-gestreamte Läufe zusätzlich `--read-timeout 600`. +- **`--stream` benutzen.** Beim Streaming setzt jeder Token den Read-Timeout + zurück; ohne Streaming muss `read_timeout` die gesamte Generierung abdecken. + Chat-`read_timeout`/`stream` sind pro Profil in der Config setzbar (Default + `read_timeout = 600`), `--read-timeout`/`--stream` auf der CLI überschreiben. + +## Coding-Prompts mit dem `ornith35b`-Profil + +Die `coding_*`-Prompts lassen sich direkt gegen das Coding-Profil testen. Es +bringt deterministischeres Sampling mit und aktiviert `stream` + hohen +`read_timeout` bereits in der Config, es sind also keine Extra-Flags nötig: + +```bash +llamacppctl --change --profile ornith35b # Coding-Modell laden +llamacppctl --chat --profile ornith35b \ + --system-prompt-profile coding \ + --prompt-file example_user_prompts/coding_05_rng_uniform.md +``` diff --git a/example_user_prompts/coding_05_rng_uniform.md b/example_user_prompts/coding_05_rng_uniform.md new file mode 100644 index 0000000..7cf68af --- /dev/null +++ b/example_user_prompts/coding_05_rng_uniform.md @@ -0,0 +1,10 @@ +Implementiere in Python eine Funktion `random_unit_float() -> float`, die eine möglichst „perfekte" gleichverteilte Zufallszahl im halboffenen Intervall [0.0, 1.0) liefert. + +Anforderungen: +- Verwende eine kryptografisch starke Entropiequelle aus der Standardbibliothek (kein `random.random()`), z. B. `secrets`/`os.urandom`. +- Nutze die volle 53-Bit-Mantisse eines `float64` und erzeuge die Zahl bias-frei (kein Modulo-Bias, keine ungleichmäßige Rundung); begründe die Wahl von genau 53 Bit. +- Halte das Intervall sauber halboffen: 0.0 ist möglich, 1.0 darf niemals herauskommen. +- Type Hints, ein knapper Docstring und keine unnötigen Abhängigkeiten. +- pytest-Tests, die abdecken: Wertebereich [0.0, 1.0) über viele Ziehungen, dass 1.0 nie auftritt, und ein einfacher Verteilungs-Check (z. B. Bucket-/Mittelwert-Test) mit begründeter Toleranz. + +Erkläre im Anschluss kurz die wichtigsten Designentscheidungen (warum 53 Bit, warum die gewählte Quelle, wie 1.0 ausgeschlossen wird).