From e7a2b8de819952c106da71b8202a3ce37f51a5c2 Mon Sep 17 00:00:00 2001 From: dschlueter Date: Fri, 18 Sep 2026 21:48:18 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20stufen-spezifisches=20Token-Limit=20f?= =?UTF-8?q?=C3=BCr=20die=20Solver-Stufe=20(LLM=5FMAX=5FTOKENS=5FSOLVER)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Die Solver-Stufe erzeugt langen Python-Code plus Reasoning. Bei komplexen Aufgaben (Portfoliomanagement: 4× je ~26k Reasoning-Tokens am 32k-Limit abgeschnitten, $0,28 für nichts) reichte das globale Limit nicht. DeepSeek V4.1 Flash unterstützt bis 393k Output-Tokens (OpenRouter). Neu: LLM_MAX_TOKENS_SOLVER (Default 65536, 0 = globales Limit gilt auch für den Solver). Wirkt nur für die Stufen 'Solver-Code' und 'Solver-Reparatur'. CLI-Option --max-tokens-solver. 438 Tests grün. --- TextToModel.py | 9 +++++++-- core/llm.py | 15 ++++++++++++--- tests/test_cli.py | 16 ++++++++++------ tests/test_llm.py | 22 ++++++++++++++++++++++ 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/TextToModel.py b/TextToModel.py index 7ef7ed2..ba171a3 100644 --- a/TextToModel.py +++ b/TextToModel.py @@ -554,6 +554,9 @@ def build_parser() -> argparse.ArgumentParser: help="Denkaufwand, nur OpenRouter (LLM_REASONING_EFFORT, Modellvorgabe)") ki.add_argument("--max-tokens", type=int, metavar="N", help=f"Antwortlänge je Aufruf (LLM_MAX_TOKENS, {llm.LLM_MAX_TOKENS})") + ki.add_argument("--max-tokens-solver", type=int, metavar="N", + help=f"Antwortlänge für die Solver-Stufe, 0 = wie --max-tokens " + f"(LLM_MAX_TOKENS_SOLVER, {llm.LLM_MAX_TOKENS_SOLVER})") ki.add_argument("--llm-timeout", type=int, metavar="SEK", help=f"Zeitlimit je Aufruf (LLM_TIMEOUT, {llm.LLM_TIMEOUT})") @@ -594,8 +597,8 @@ def _check_args(parser: argparse.ArgumentParser, args: argparse.Namespace) -> No ("--solver-retries", args.solver_retries)): if wert < 0: parser.error(f"{name} darf nicht negativ sein") - for name, wert in (("--max-tokens", args.max_tokens), ("--llm-timeout", args.llm_timeout), - ("--solver-timeout", args.solver_timeout)): + for name, wert in (("--max-tokens", args.max_tokens), ("--max-tokens-solver", args.max_tokens_solver), + ("--llm-timeout", args.llm_timeout), ("--solver-timeout", args.solver_timeout)): if wert is not None and wert <= 0: parser.error(f"{name} muss positiv sein") eingaben = [args.aufgabe if args.aufgabe != "-" else None, args.von[1] if args.von else None, @@ -620,6 +623,8 @@ def apply_settings(args: argparse.Namespace) -> None: llm.REASONING_EFFORT = args.reasoning if args.max_tokens: llm.LLM_MAX_TOKENS = args.max_tokens + if args.max_tokens_solver is not None: + llm.LLM_MAX_TOKENS_SOLVER = args.max_tokens_solver if args.llm_timeout: llm.LLM_TIMEOUT = args.llm_timeout if args.solver_timeout: diff --git a/core/llm.py b/core/llm.py index 726e59f..488d4fe 100644 --- a/core/llm.py +++ b/core/llm.py @@ -47,6 +47,11 @@ LLM_TIMEOUT = int(os.environ.get("LLM_TIMEOUT", "420")) # Sekunden je Aufruf # Ohne Vorgabe schneiden manche OpenRouter-Anbieter die Antwort ab (beobachtet: abgebrochene # Modellspezifikation in einer Korrekturrunde). Großzügig setzen; bezahlt werden nur erzeugte Tokens. LLM_MAX_TOKENS = int(os.environ.get("LLM_MAX_TOKENS", "32000")) +# Die Solver-Stufe erzeugt langen Python-Code plus Reasoning – bei komplexen Aufgaben (beobachtet: +# Portfoliomanagement, 4× je ~26k Reasoning-Tokens am 32k-Limit abgeschnitten) reicht das globale +# Limit nicht. DeepSeek V4.1 Flash unterstützt bis 393k Output-Tokens (OpenRouter), wir setzen die +# Solver-Stufe bewusst höher. 0 = das globale Limit gilt auch für den Solver. +LLM_MAX_TOKENS_SOLVER = int(os.environ.get("LLM_MAX_TOKENS_SOLVER", "65536")) # Denkaufwand (OpenRouter): "low" | "medium" | "high"; leer = Modellvorgabe. # Reasoning-Tokens machen den Großteil der Laufzeit aus (gemessen: 45.000 von 73.000 Output-Tokens). REASONING_EFFORT = os.environ.get("LLM_REASONING_EFFORT", "").strip().lower() @@ -159,10 +164,14 @@ class LLMSession: """Stufen mit abweichendem Modell – für die Fortschrittsausgabe.""" return {step: m for step, m in self.step_models.items() if m != self.model} - def _kwargs(self, model: str, response_model, messages) -> dict: + def _kwargs(self, model: str, response_model, messages, step: str = "") -> dict: kwargs = {"model": model, "response_model": response_model, "messages": messages} if provider_for(model) == "openrouter": - kwargs["max_tokens"] = LLM_MAX_TOKENS + # Solver-Stufe bekommt ein eigenes, höheres Limit (siehe LLM_MAX_TOKENS_SOLVER). + if step in ("Solver-Code", "Solver-Reparatur") and LLM_MAX_TOKENS_SOLVER > 0: + kwargs["max_tokens"] = LLM_MAX_TOKENS_SOLVER + else: + kwargs["max_tokens"] = LLM_MAX_TOKENS # nur Anbieter wählen, die strukturierte Ausgabe (response_format) wirklich unterstützen anbieter = {"require_parameters": True, "preferred_min_throughput": {"p50": OPENROUTER_MIN_THROUGHPUT}} if self.ignored_providers: @@ -177,7 +186,7 @@ class LLMSession: provider = provider_for(model) waits = list(RETRY_WAITS) while True: - kwargs = self._kwargs(model, response_model, messages) + kwargs = self._kwargs(model, response_model, messages, step) started = time.monotonic() try: result, completion = self._clients[provider].chat.completions.create_with_completion(**kwargs) diff --git a/tests/test_cli.py b/tests/test_cli.py index 277e1f8..f04f53c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,7 +14,8 @@ from test_resolve import gespeichertes_ergebnis def einstellungen_zuruecksetzen(monkeypatch): # main() setzt Modulkonstanten; monkeypatch stellt sie nach jedem Test wieder her for modul, name in ((TextToModel, "UNDERSTAND_RETRIES"), (TextToModel, "INFEASIBLE_RECHECK"), - (llm, "REASONING_EFFORT"), (llm, "LLM_MAX_TOKENS"), (llm, "LLM_TIMEOUT"), + (llm, "REASONING_EFFORT"), (llm, "LLM_MAX_TOKENS"), (llm, "LLM_MAX_TOKENS_SOLVER"), + (llm, "LLM_TIMEOUT"), (solver_stage, "SOLVER_TIMEOUT"), (solver_stage, "SANDBOX")): monkeypatch.setattr(modul, name, getattr(modul, name)) @@ -101,7 +102,8 @@ def test_options_reach_pipeline_and_modules(tmp_path, monkeypatch): monkeypatch.setattr(TextToModel, "run_pipeline", fake_pipeline) code = TextToModel.main([str(aufgabe), "-m", "x/y", "--max-revisions", "0", "--solver-retries", "4", "--max-restarts", "0", "--understand-retries", "-1", "--infeasible-recheck", "0", - "--reasoning", "low", "--max-tokens", "1000", "--llm-timeout", "30", + "--reasoning", "low", "--max-tokens", "1000", "--max-tokens-solver", "2000", + "--llm-timeout", "30", "--solver-timeout", "90", "--sandbox", "bwrap", "-o", str(tmp_path / "e.json")]) assert code == TextToModel.EXIT_REJECTED assert gesehen["task"].startswith("Maximiere") and gesehen["task_dir"] == tmp_path @@ -109,16 +111,18 @@ def test_options_reach_pipeline_and_modules(tmp_path, monkeypatch): == ("x/y", 0, 4, 0) assert gesehen["solve_model"] is True assert (TextToModel.UNDERSTAND_RETRIES, TextToModel.INFEASIBLE_RECHECK) == (-1, 0) - assert (llm.REASONING_EFFORT, llm.LLM_MAX_TOKENS, llm.LLM_TIMEOUT) == ("low", 1000, 30) + assert (llm.REASONING_EFFORT, llm.LLM_MAX_TOKENS, llm.LLM_MAX_TOKENS_SOLVER, llm.LLM_TIMEOUT) \ + == ("low", 1000, 2000, 30) assert (solver_stage.SOLVER_TIMEOUT, solver_stage.SANDBOX) == (90, "bwrap") assert "90" in solver_stage.solver_prompt() # Zeitbudget im Prompt folgt der Option def test_without_options_the_defaults_stay(tmp_path, monkeypatch): - vorher = (llm.LLM_MAX_TOKENS, solver_stage.SOLVER_TIMEOUT, solver_stage.SANDBOX, TextToModel.UNDERSTAND_RETRIES) + vorher = (llm.LLM_MAX_TOKENS, llm.LLM_MAX_TOKENS_SOLVER, solver_stage.SOLVER_TIMEOUT, + solver_stage.SANDBOX, TextToModel.UNDERSTAND_RETRIES) monkeypatch.setattr(TextToModel, "run_pipeline", lambda task, **kw: PipelineResult(task=task, reliable=True)) aufgabe = tmp_path / "a.txt" aufgabe.write_text("Minimiere x.", encoding="utf-8") TextToModel.main([str(aufgabe), "-o", str(tmp_path / "e.json")]) - assert (llm.LLM_MAX_TOKENS, solver_stage.SOLVER_TIMEOUT, solver_stage.SANDBOX, - TextToModel.UNDERSTAND_RETRIES) == vorher + assert (llm.LLM_MAX_TOKENS, llm.LLM_MAX_TOKENS_SOLVER, solver_stage.SOLVER_TIMEOUT, + solver_stage.SANDBOX, TextToModel.UNDERSTAND_RETRIES) == vorher diff --git a/tests/test_llm.py b/tests/test_llm.py index 6eff22a..d63f835 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -86,6 +86,28 @@ def test_max_tokens_is_sent_to_openrouter(): assert completions["openrouter"].kwargs["max_tokens"] == llm.LLM_MAX_TOKENS >= 16000 +def test_solver_step_uses_higher_max_tokens(): + # beobachtet am Portfoliomanagement: 4× je ~26k Reasoning-Tokens am 32k-Limit abgeschnitten. + # DeepSeek V4.1 Flash unterstützt bis 393k Output-Tokens (OpenRouter) – die Solver-Stufe bekommt + # ein eigenes, höheres Limit, die übrigen Stufen bleiben beim globalen. + s_solver, c_solver = fake_session() + s_solver.create(str, [{"role": "user", "content": "x"}], step="Solver-Code") + assert c_solver["openrouter"].kwargs["max_tokens"] == llm.LLM_MAX_TOKENS_SOLVER > llm.LLM_MAX_TOKENS + s_repair, c_repair = fake_session() + s_repair.create(str, [{"role": "user", "content": "y"}], step="Solver-Reparatur") + assert c_repair["openrouter"].kwargs["max_tokens"] == llm.LLM_MAX_TOKENS_SOLVER + s_extra, c_extra = fake_session() + s_extra.create(str, [{"role": "user", "content": "z"}], step="Extraktion") + assert c_extra["openrouter"].kwargs["max_tokens"] == llm.LLM_MAX_TOKENS + + +def test_solver_max_tokens_zero_falls_back_to_global(monkeypatch): + monkeypatch.setattr(llm, "LLM_MAX_TOKENS_SOLVER", 0) + session, completions = fake_session() + session.create(str, [{"role": "user", "content": "x"}], step="Solver-Code") + assert completions["openrouter"].kwargs["max_tokens"] == llm.LLM_MAX_TOKENS + + # --- Modell je Pipeline-Stufe (LLM_MODEL_EXTRACTION, LLM_MODEL_SOLVER, ...) --- def test_step_model_falls_back_to_session_model(monkeypatch):