diff --git a/CHANGELOG.md b/CHANGELOG.md index 1463e60..5fd3f4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,16 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Added -- Per-profile chat streaming and chat-request timeouts: `stream`, `read_timeout` - and `connect_timeout` are now resolvable in `[default]`/`[model.]` - (CLI `--stream`/`--read-timeout`/`--connect-timeout` still override). - -### Changed -- The chat request now honors the configured `(connect, read)` timeout instead - of a hard-coded 30 s; the default `read_timeout` is 600 s so slow reasoning - models are no longer cut off mid-generation. - ## [0.1.0] - 2026-07-07 Initial release. Replaces the previous collection of shell scripts diff --git a/example_user_prompts/README.md b/example_user_prompts/README.md index 8bee342..3287487 100644 --- a/example_user_prompts/README.md +++ b/example_user_prompts/README.md @@ -13,7 +13,7 @@ Referenzmodell (zunächst): das Default-Modell aus `llama.cpp.config` |---|---| | `system_prompt_prosa.md` | `prosa_01_werkstatt.md`, `prosa_02_dystopie.md`, `prosa_03_essay.md`, `prosa_04_dialog.md` | | `system_prompt_reden.md` | `reden_01_abifeier.md`, `reden_02_windkraft.md`, `reden_03_umstrukturierung.md`, `reden_04_gedenken.md` | -| `system_prompt_coding.md` | `coding_01_jsonl_validator.md`, `coding_02_refactor.md`, `coding_03_ts_retry.md`, `coding_04_performance.md`, `coding_05_rng_uniform.md` | +| `system_prompt_coding.md` | `coding_01_jsonl_validator.md`, `coding_02_refactor.md`, `coding_03_ts_retry.md`, `coding_04_performance.md` | Jeder User-Prompt fordert gezielt Eigenschaften heraus, die der jeweilige System-Prompt verspricht (z. B. „show, don't tell" und Schlussbild bei Prosa; @@ -37,20 +37,7 @@ Hinweise: (`--max-tokens`) muss **Denk- plus Ausgabe-Tokens** abdecken — für Prosa/Reden großzügig wählen (z. B. 6000–10000), sonst bricht die sichtbare Antwort ab oder bleibt leer. -- **`--stream` benutzen.** Beim Streaming setzt jeder Token den Read-Timeout - zurück; ohne Streaming muss `read_timeout` die gesamte Generierung abdecken. - Chat-`read_timeout`/`stream` sind pro Profil in der Config setzbar (Default - `read_timeout = 600`), `--read-timeout`/`--stream` auf der CLI überschreiben. - -## Coding-Prompts mit dem `ornith35b`-Profil - -Die `coding_*`-Prompts lassen sich direkt gegen das Coding-Profil testen. Es -bringt deterministischeres Sampling mit und aktiviert `stream` + hohen -`read_timeout` bereits in der Config, es sind also keine Extra-Flags nötig: - -```bash -llamacppctl --change --profile ornith35b # Coding-Modell laden -llamacppctl --chat --profile ornith35b \ - --system-prompt-profile coding \ - --prompt-file example_user_prompts/coding_05_rng_uniform.md -``` +- **`--stream` benutzen.** Ohne Streaming wartet das Tool die komplette Antwort + in einem einzigen Read ab und läuft beim langen Reasoning-Output in den + Default-`--read-timeout` (30 s). Beim Streaming setzt jeder Token den Timeout + zurück. Für lange nicht-gestreamte Läufe zusätzlich `--read-timeout 600`. diff --git a/example_user_prompts/coding_05_rng_uniform.md b/example_user_prompts/coding_05_rng_uniform.md deleted file mode 100644 index 7cf68af..0000000 --- a/example_user_prompts/coding_05_rng_uniform.md +++ /dev/null @@ -1,10 +0,0 @@ -Implementiere in Python eine Funktion `random_unit_float() -> float`, die eine möglichst „perfekte" gleichverteilte Zufallszahl im halboffenen Intervall [0.0, 1.0) liefert. - -Anforderungen: -- Verwende eine kryptografisch starke Entropiequelle aus der Standardbibliothek (kein `random.random()`), z. B. `secrets`/`os.urandom`. -- Nutze die volle 53-Bit-Mantisse eines `float64` und erzeuge die Zahl bias-frei (kein Modulo-Bias, keine ungleichmäßige Rundung); begründe die Wahl von genau 53 Bit. -- Halte das Intervall sauber halboffen: 0.0 ist möglich, 1.0 darf niemals herauskommen. -- Type Hints, ein knapper Docstring und keine unnötigen Abhängigkeiten. -- pytest-Tests, die abdecken: Wertebereich [0.0, 1.0) über viele Ziehungen, dass 1.0 nie auftritt, und ein einfacher Verteilungs-Check (z. B. Bucket-/Mittelwert-Test) mit begründeter Toleranz. - -Erkläre im Anschluss kurz die wichtigsten Designentscheidungen (warum 53 Bit, warum die gewählte Quelle, wie 1.0 ausgeschlossen wird). diff --git a/llama.cpp.config.example b/llama.cpp.config.example index b411535..628da02 100644 --- a/llama.cpp.config.example +++ b/llama.cpp.config.example @@ -62,14 +62,6 @@ api_key = max_tokens = 2048 # chat_temperature leer lassen -> die Server-Temperatur (temp) gilt. chat_temperature = -# Chat-Antwort standardmäßig token-weise streamen (wie --stream). Für langsame -# Reasoning-Modelle angenehm, weil sofort Ausgabe erscheint. -stream = false -# (connect, read)-Timeout des Chat-Requests in Sekunden. read_timeout muss die -# volle Generierungszeit abdecken; Reasoning-Modelle brauchen viel -> hoch -# setzen. --read-timeout/--connect-timeout auf der CLI überschreiben das. -read_timeout = 600 -connect_timeout = 10 # Example second model profile, pinned to the second GPU (e.g. RTX 3090 #2) # with its own port and container name so it can run alongside [default]. diff --git a/src/llamacppctl/actions.py b/src/llamacppctl/actions.py index 9011338..f8cd8ad 100644 --- a/src/llamacppctl/actions.py +++ b/src/llamacppctl/actions.py @@ -233,8 +233,7 @@ def do_chat(cfg: ServerConfig, prompt_cfg: PromptConfig, args) -> int: print("--chat requires a user prompt", file=sys.stderr) return EXIT_GENERAL - # Streaming can be requested via --stream OR enabled per profile in config. - if getattr(args, "stream", False) or getattr(prompt_cfg, "stream", False): + if getattr(args, "stream", False): return _do_chat_stream(cfg, prompt_cfg, args) try: diff --git a/src/llamacppctl/cli.py b/src/llamacppctl/cli.py index 11ed47f..2a99f98 100644 --- a/src/llamacppctl/cli.py +++ b/src/llamacppctl/cli.py @@ -103,11 +103,8 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--no-allow-symlinks", dest="allow_symlinks", action="store_false") p.add_argument("--max-input-bytes", type=int, default=1_048_576) p.add_argument("--url-allow-host", action="append", default=[]) - # Default None so a config-provided value can win; a fallback is applied at - # the point of use (URL fetch: 3s/10s; chat request: profile read_timeout). - # When set, these also override the chat request's (connect, read) timeout. - p.add_argument("--connect-timeout", type=float, default=None) - p.add_argument("--read-timeout", type=float, default=None) + p.add_argument("--connect-timeout", type=float, default=3.0) + p.add_argument("--read-timeout", type=float, default=10.0) # --- Chat request parameters (for --chat and the --start reply) --- p.add_argument( @@ -169,9 +166,7 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> if args.max_tokens is not None and args.max_tokens <= 0: parser.error("--max-tokens must be > 0") - if (args.connect_timeout is not None and args.connect_timeout <= 0) or ( - args.read_timeout is not None and args.read_timeout <= 0 - ): + if args.connect_timeout <= 0 or args.read_timeout <= 0: parser.error("timeouts must be > 0") if args.allow_private_url and args.url_allow_host: diff --git a/src/llamacppctl/config.py b/src/llamacppctl/config.py index 684a6e9..301ba9a 100644 --- a/src/llamacppctl/config.py +++ b/src/llamacppctl/config.py @@ -248,29 +248,11 @@ def resolve_effective_config(args) -> tuple: if getattr(args, "chat_temp", None) is not None: chat_temp = args.chat_temp - # Streaming: enabled per profile via `stream = true`, or with --stream. - stream = str(merged.get("stream", "")).strip().lower() in TRUE_STRINGS - if getattr(args, "stream", False): - stream = True - - # Chat request timeouts: config value first, CLI flag overrides. These are - # the (connect, read) timeouts of the chat POST; read_timeout must cover the - # model's full generation time (raise it for slow reasoning models). - read_timeout = float(merged.get("read_timeout") or 600.0) - if getattr(args, "read_timeout", None) is not None: - read_timeout = args.read_timeout - connect_timeout = float(merged.get("connect_timeout") or 10.0) - if getattr(args, "connect_timeout", None) is not None: - connect_timeout = args.connect_timeout - prompt_cfg = PromptConfig( system_prompt=system_prompt, user_prompt=getattr(args, "_resolved_user_prompt", None), max_tokens=max_tokens, temperature=chat_temp, - stream=stream, - connect_timeout=connect_timeout, - read_timeout=read_timeout, ) return server_cfg, prompt_cfg diff --git a/src/llamacppctl/http_ops.py b/src/llamacppctl/http_ops.py index 420c0fa..f414914 100644 --- a/src/llamacppctl/http_ops.py +++ b/src/llamacppctl/http_ops.py @@ -88,21 +88,18 @@ def chat_completion(cfg: ServerConfig, prompt_cfg: PromptConfig) -> requests.Res def chat_completion_text( - cfg: ServerConfig, prompt_cfg: PromptConfig, timeout: Optional[tuple] = None + cfg: ServerConfig, prompt_cfg: PromptConfig, timeout: float = 30.0 ) -> ChatReply: """High-level chat helper for --chat and the --start/--check post-checks. Returns a ChatReply (content + finish_reason), raising HttpError on - transport, HTTP status, JSON decoding, or malformed-response failures. - - The (connect, read) timeout defaults to the prompt profile's configured - values so slow reasoning models are not cut off mid-generation.""" + transport, HTTP status, JSON decoding, or malformed-response failures.""" try: resp = requests.post( chat_url(cfg), json=_chat_payload(cfg, prompt_cfg), headers=_auth_headers(cfg), - timeout=timeout or (prompt_cfg.connect_timeout, prompt_cfg.read_timeout), + timeout=timeout, ) except requests.RequestException as exc: raise HttpError(f"chat completion request failed: {exc}") from exc @@ -140,7 +137,7 @@ def stream_chat( json=payload, headers=_auth_headers(cfg), stream=True, - timeout=timeout or (prompt_cfg.connect_timeout, prompt_cfg.read_timeout), + timeout=timeout or (10.0, 600.0), ) except requests.RequestException as exc: raise HttpError(f"chat completion request failed: {exc}") from exc diff --git a/src/llamacppctl/main.py b/src/llamacppctl/main.py index b633b96..3bf16c1 100644 --- a/src/llamacppctl/main.py +++ b/src/llamacppctl/main.py @@ -24,8 +24,8 @@ from .prompt_io import InputPolicy, PromptSourceError, load_prompt_source, resol def _build_input_policy(args) -> InputPolicy: return InputPolicy( max_input_bytes=args.max_input_bytes, - connect_timeout=args.connect_timeout if args.connect_timeout is not None else 3.0, - read_timeout=args.read_timeout if args.read_timeout is not None else 10.0, + connect_timeout=args.connect_timeout, + read_timeout=args.read_timeout, allow_insecure_http=args.allow_insecure_http, allow_private_url=args.allow_private_url, allow_ip_host=args.allow_ip_host, diff --git a/src/llamacppctl/schema.py b/src/llamacppctl/schema.py index 13018c0..ab057d7 100644 --- a/src/llamacppctl/schema.py +++ b/src/llamacppctl/schema.py @@ -73,12 +73,6 @@ class PromptConfig: # None => omit from the request so the server's configured --temp applies. temperature: Optional[float] = None stream: bool = False - # HTTP timeouts for the chat request itself (seconds). read_timeout must - # cover the model's full generation time: reasoning models can "think" for - # a long time before the first/last token, so the default is generous. Both - # are configurable per profile (read_timeout/connect_timeout in the config). - connect_timeout: float = 10.0 - read_timeout: float = 600.0 @dataclass diff --git a/tests/test_config.py b/tests/test_config.py index 38ddb2b..1b162dc 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -258,33 +258,3 @@ def test_chat_params_cli_overrides_config(tmp_path): _, prompt_cfg = resolve_effective_config(args) assert prompt_cfg.max_tokens == 512 assert prompt_cfg.temperature == 0.1 - - -def test_stream_and_timeouts_default(tmp_path): - cfg_path = _write_config(tmp_path) - _, prompt_cfg = resolve_effective_config(_parse(["--start", "--config", str(cfg_path)])) - assert prompt_cfg.stream is False - assert prompt_cfg.read_timeout == 600.0 # generous default for reasoning models - assert prompt_cfg.connect_timeout == 10.0 - - -def test_stream_and_read_timeout_from_config(tmp_path): - cfg = CONFIG_BASIC.replace( - "poll_interval = 2", - "poll_interval = 2\nstream = true\nread_timeout = 900\nconnect_timeout = 5", - ) - cfg_path = _write_config(tmp_path, cfg) - _, prompt_cfg = resolve_effective_config(_parse(["--start", "--config", str(cfg_path)])) - assert prompt_cfg.stream is True - assert prompt_cfg.read_timeout == 900.0 - assert prompt_cfg.connect_timeout == 5.0 - - -def test_stream_and_read_timeout_cli_overrides_config(tmp_path): - cfg = CONFIG_BASIC.replace("poll_interval = 2", "poll_interval = 2\nread_timeout = 900") - cfg_path = _write_config(tmp_path, cfg) - # --stream forces streaming on; --read-timeout overrides the config value. - args = _parse(["--chat", "-p", "hi", "--config", str(cfg_path), "--stream", "--read-timeout", "42"]) - _, prompt_cfg = resolve_effective_config(args) - assert prompt_cfg.stream is True - assert prompt_cfg.read_timeout == 42.0