dschlueter
e7a2b8de81
Die Solver-Stufe erzeugt langen Python-Code plus Reasoning. Bei komplexen Aufgaben (Portfoliomanagement: 4× je ~26k Reasoning-Tokens am 32k-Limit abgeschnitten, $0,28 für nichts) reichte das globale Limit nicht. DeepSeek V4.1 Flash unterstützt bis 393k Output-Tokens (OpenRouter). Neu: LLM_MAX_TOKENS_SOLVER (Default 65536, 0 = globales Limit gilt auch für den Solver). Wirkt nur für die Stufen 'Solver-Code' und 'Solver-Reparatur'. CLI-Option --max-tokens-solver. 438 Tests grün.
128 lines
6.3 KiB
Python
128 lines
6.3 KiB
Python
"""CLI `TextToModel.main`: Optionen, Defaults, Ausgabe und Exit-Codes – ohne LLM."""
|
||
import json
|
||
|
||
import pytest
|
||
|
||
import TextToModel
|
||
from core import llm
|
||
from core.model_schema import PipelineResult
|
||
from solving import solver_stage
|
||
from test_resolve import gespeichertes_ergebnis
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def einstellungen_zuruecksetzen(monkeypatch):
|
||
# main() setzt Modulkonstanten; monkeypatch stellt sie nach jedem Test wieder her
|
||
for modul, name in ((TextToModel, "UNDERSTAND_RETRIES"), (TextToModel, "INFEASIBLE_RECHECK"),
|
||
(llm, "REASONING_EFFORT"), (llm, "LLM_MAX_TOKENS"), (llm, "LLM_MAX_TOKENS_SOLVER"),
|
||
(llm, "LLM_TIMEOUT"),
|
||
(solver_stage, "SOLVER_TIMEOUT"), (solver_stage, "SANDBOX")):
|
||
monkeypatch.setattr(modul, name, getattr(modul, name))
|
||
|
||
|
||
def aufruffehler(argv, capsys) -> str:
|
||
with pytest.raises(SystemExit) as exc:
|
||
TextToModel.main(argv)
|
||
assert exc.value.code == TextToModel.EXIT_USAGE # nicht 2 – das hieße „unzuverlässig“
|
||
return capsys.readouterr().err
|
||
|
||
|
||
def test_resolve_writes_json_report_and_code_with_prefix(tmp_path):
|
||
ergebnis = gespeichertes_ergebnis(tmp_path)
|
||
praefix = tmp_path / "neu" / "lauf" # Ordner existiert noch nicht
|
||
assert TextToModel.main(["--resolve", str(ergebnis), "-a", str(praefix)]) == TextToModel.EXIT_OK
|
||
neu = PipelineResult.model_validate_json((tmp_path / "neu" / "lauf.json").read_text(encoding="utf-8"))
|
||
assert neu.solver_run.objective_value == pytest.approx(9)
|
||
assert (tmp_path / "neu" / "lauf.html").exists() and (tmp_path / "neu" / "lauf.py").exists()
|
||
|
||
|
||
def test_explicit_output_wins_over_prefix(tmp_path):
|
||
ergebnis = gespeichertes_ergebnis(tmp_path)
|
||
TextToModel.main(["--resolve", str(ergebnis), "-a", str(tmp_path / "p"), "--report", str(tmp_path / "r.md")])
|
||
assert (tmp_path / "r.md").exists() and not (tmp_path / "p.html").exists()
|
||
|
||
|
||
def test_json_goes_to_redirected_stdout(tmp_path, capsys):
|
||
ergebnis = gespeichertes_ergebnis(tmp_path)
|
||
daten = tmp_path / "daten.json"
|
||
daten.write_text(json.dumps({"bedarf": 5}), encoding="utf-8")
|
||
assert TextToModel.main(["--resolve", str(ergebnis), "--data", str(daten)]) == TextToModel.EXIT_OK
|
||
assert json.loads(capsys.readouterr().out)["solver_run"]["objective_value"] == pytest.approx(12)
|
||
|
||
|
||
def test_terminal_gets_a_summary_instead_of_json(tmp_path, capsys, monkeypatch):
|
||
monkeypatch.setattr("sys.stdout.isatty", lambda: True)
|
||
TextToModel.main(["--resolve", str(gespeichertes_ergebnis(tmp_path))])
|
||
out = capsys.readouterr().out
|
||
assert "Zielwert: 9" in out and "Zuverlässig." in out and not out.lstrip().startswith("{")
|
||
|
||
|
||
def test_unreliable_result_exits_with_2(tmp_path):
|
||
daten = tmp_path / "daten.json"
|
||
daten.write_text(json.dumps({"kosten": "keine Zahl"}), encoding="utf-8")
|
||
code = TextToModel.main(["--resolve", str(gespeichertes_ergebnis(tmp_path)), "--data", str(daten),
|
||
"-o", str(tmp_path / "x.json")])
|
||
assert code == TextToModel.EXIT_UNRELIABLE
|
||
|
||
|
||
def test_broken_result_file_is_an_error_not_a_traceback(tmp_path):
|
||
kaputt = tmp_path / "kaputt.json"
|
||
kaputt.write_text("{}", encoding="utf-8")
|
||
assert TextToModel.main(["--resolve", str(kaputt)]) == TextToModel.EXIT_ERROR
|
||
|
||
|
||
@pytest.mark.parametrize("argv, meldung", [
|
||
(["--data", "d.json"], "--data wirkt nur"),
|
||
(["a.txt", "--report", "r.pdf"], "Endung"),
|
||
(["a.txt", "--no-solve", "--code", "c.py"], "--no-solve"),
|
||
(["--von", "extraktion", "e.json"], "Stufe"),
|
||
(["--von", "solver", "e.json", "--no-solve"], "not allowed"),
|
||
(["--max-revisions", "-1"], "negativ"),
|
||
(["--solver-timeout", "0"], "positiv"),
|
||
(["fehlt.txt"], "Datei nicht gefunden"),
|
||
])
|
||
def test_wrong_calls_are_usage_errors(argv, meldung, capsys):
|
||
assert meldung in aufruffehler(argv, capsys)
|
||
|
||
|
||
def test_task_file_with_resolve_is_rejected(tmp_path, capsys):
|
||
ergebnis = gespeichertes_ergebnis(tmp_path)
|
||
assert "keine Aufgabendatei" in aufruffehler([str(ergebnis), "--resolve", str(ergebnis)], capsys)
|
||
|
||
|
||
def test_options_reach_pipeline_and_modules(tmp_path, monkeypatch):
|
||
aufgabe = tmp_path / "aufgabe.md"
|
||
aufgabe.write_text("Maximiere x unter x ≤ 3.", encoding="utf-8")
|
||
gesehen = {}
|
||
|
||
def fake_pipeline(task, **kwargs):
|
||
gesehen.update(kwargs, task=task)
|
||
return PipelineResult(task=task, rejected=True, reliable=False, reliability_issues=["keine Aufgabe"])
|
||
|
||
monkeypatch.setattr(TextToModel, "run_pipeline", fake_pipeline)
|
||
code = TextToModel.main([str(aufgabe), "-m", "x/y", "--max-revisions", "0", "--solver-retries", "4",
|
||
"--max-restarts", "0", "--understand-retries", "-1", "--infeasible-recheck", "0",
|
||
"--reasoning", "low", "--max-tokens", "1000", "--max-tokens-solver", "2000",
|
||
"--llm-timeout", "30",
|
||
"--solver-timeout", "90", "--sandbox", "bwrap", "-o", str(tmp_path / "e.json")])
|
||
assert code == TextToModel.EXIT_REJECTED
|
||
assert gesehen["task"].startswith("Maximiere") and gesehen["task_dir"] == tmp_path
|
||
assert (gesehen["model"], gesehen["max_revisions"], gesehen["solver_retries"], gesehen["max_restarts"]) \
|
||
== ("x/y", 0, 4, 0)
|
||
assert gesehen["solve_model"] is True
|
||
assert (TextToModel.UNDERSTAND_RETRIES, TextToModel.INFEASIBLE_RECHECK) == (-1, 0)
|
||
assert (llm.REASONING_EFFORT, llm.LLM_MAX_TOKENS, llm.LLM_MAX_TOKENS_SOLVER, llm.LLM_TIMEOUT) \
|
||
== ("low", 1000, 2000, 30)
|
||
assert (solver_stage.SOLVER_TIMEOUT, solver_stage.SANDBOX) == (90, "bwrap")
|
||
assert "90" in solver_stage.solver_prompt() # Zeitbudget im Prompt folgt der Option
|
||
|
||
|
||
def test_without_options_the_defaults_stay(tmp_path, monkeypatch):
|
||
vorher = (llm.LLM_MAX_TOKENS, llm.LLM_MAX_TOKENS_SOLVER, solver_stage.SOLVER_TIMEOUT,
|
||
solver_stage.SANDBOX, TextToModel.UNDERSTAND_RETRIES)
|
||
monkeypatch.setattr(TextToModel, "run_pipeline", lambda task, **kw: PipelineResult(task=task, reliable=True))
|
||
aufgabe = tmp_path / "a.txt"
|
||
aufgabe.write_text("Minimiere x.", encoding="utf-8")
|
||
TextToModel.main([str(aufgabe), "-o", str(tmp_path / "e.json")])
|
||
assert (llm.LLM_MAX_TOKENS, llm.LLM_MAX_TOKENS_SOLVER, solver_stage.SOLVER_TIMEOUT,
|
||
solver_stage.SANDBOX, TextToModel.UNDERSTAND_RETRIES) == vorher
|