dschlueter
b74445b8d6
Übersetzt natürlichsprachige Optimierungsaufgaben (Deutsch) per LLM in eine geprüfte Modellspezifikation, erzeugt Solver-Code und führt ihn in einer bubblewrap-Sandbox aus. Deterministische Prüfungen ohne LLM, Report als Markdown/HTML, Benchmark mit 19 verifizierten Fällen. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
545 lines
26 KiB
Python
545 lines
26 KiB
Python
"""run_pipeline ohne LLM: Neustart bei Fremdproblem und Kennzeichnung unzuverlässiger Ergebnisse."""
|
||
from benchmark import result_correct
|
||
from model_schema import (
|
||
AssumptionReview,
|
||
LLMUsage,
|
||
OptimizationModelSpecification,
|
||
SolverCode,
|
||
SolverStatus,
|
||
SubQuestion,
|
||
TaskUnderstanding,
|
||
)
|
||
from TextToModel import run_pipeline
|
||
from test_model_checks import make_spec
|
||
from test_review import make_review
|
||
from test_script_export import STRUCTURED_CODE
|
||
|
||
|
||
# Standardverstaendnis: eine Frage, keine Zahlen - reicht fuer Tests, die das Verstehen nicht pruefen
|
||
EINFACHES_VERSTAENDNIS = TaskUnderstanding(
|
||
is_optimization_task=True, reason="-", facts=[], ignored_numbers=[],
|
||
sub_questions=[SubQuestion(id="1", quote="-", kind="rechnen", needs_solver=True)])
|
||
|
||
|
||
class FakeSession:
|
||
"""Liefert vorgegebene Antworten je Antwort-Typ in fester Reihenfolge."""
|
||
|
||
def __init__(self, specs, reviews, codes=(), understandings=None):
|
||
self.queues = {
|
||
OptimizationModelSpecification: list(specs),
|
||
AssumptionReview: list(reviews),
|
||
SolverCode: list(codes),
|
||
TaskUnderstanding: list(understandings if understandings is not None else []),
|
||
}
|
||
self.usage = LLMUsage(model="fake", provider="fake")
|
||
self.calls = {kind: 0 for kind in self.queues}
|
||
self.steps = []
|
||
self.tasks = []
|
||
self.letzter_prompt = ""
|
||
|
||
def overrides(self):
|
||
return {}
|
||
|
||
def create(self, response_model, messages, step="unbenannt"):
|
||
self.letzter_prompt = messages[-1]["content"]
|
||
if step == "Extraktion":
|
||
self.tasks.append(messages[-1]["content"])
|
||
self.calls[response_model] += 1
|
||
self.usage.calls += 1
|
||
self.steps.append(step)
|
||
warteschlange = self.queues[response_model]
|
||
if response_model is TaskUnderstanding and not warteschlange:
|
||
return EINFACHES_VERSTAENDNIS # Standard, solange ein Test nichts anderes vorgibt
|
||
return warteschlange.pop(0)
|
||
|
||
|
||
HALLUCINATED = make_spec(executive_summary="Produktionsplanung mit Drehen und Fräsen") # Fremdproblem
|
||
GOOD = make_spec(executive_summary="Rohstoffeinkauf mit minimalen Kosten")
|
||
MISMATCH = make_review(matches=False)
|
||
OK = make_review()
|
||
|
||
|
||
def test_clean_first_attempt_needs_no_restart():
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=STRUCTURED_CODE)])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=2, max_restarts=1)
|
||
assert result.reliable and result.restarts == 0 and result.revisions == 0
|
||
assert result.solver_run.status == SolverStatus.OPTIMAL
|
||
|
||
|
||
def test_restart_recovers_from_hallucinated_problem():
|
||
# Versuch 1: Fremdproblem, beide Korrekturrunden erfolglos; Neustart liefert das richtige Modell
|
||
session = FakeSession(
|
||
specs=[HALLUCINATED, HALLUCINATED, HALLUCINATED, GOOD],
|
||
reviews=[MISMATCH, MISMATCH, MISMATCH, OK],
|
||
codes=[SolverCode(code=STRUCTURED_CODE)],
|
||
)
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=2, max_restarts=1)
|
||
assert result.restarts == 1
|
||
assert result.reliable and result.reliability_issues == []
|
||
assert result.specification.executive_summary == GOOD.executive_summary
|
||
assert result.solver_run.status == SolverStatus.OPTIMAL and result.solver_run.objective_value == 9
|
||
|
||
|
||
def test_persistent_mismatch_is_unreliable_and_not_solved():
|
||
# beobachtet im Benchmark: Fremdproblem wurde als "optimal, 2550" ausgegeben
|
||
session = FakeSession(specs=[HALLUCINATED] * 6, reviews=[MISMATCH] * 6)
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=2, max_restarts=1)
|
||
assert result.restarts == 1
|
||
assert not result.reliable
|
||
assert any("Zielfunktion passt nicht" in issue for issue in result.reliability_issues)
|
||
assert result.solver_run is None and result.solver_code is None
|
||
assert session.calls[SolverCode] == 0 # kein Solver-Code für ein falsches Problem
|
||
assert not result_correct(result.solver_run, {"objective_value": 9, "tolerance": 1e-6})
|
||
|
||
|
||
def test_better_earlier_attempt_wins_over_worse_restart():
|
||
# Neustart ist schlechter (Zielgröße falsch) als Versuch 1 (nur Check-Error) -> Versuch 1 verwenden
|
||
stack_error = make_spec(executive_summary="richtig, aber Stack falsch", problem_class="Quadratic Programming (QP)")
|
||
session = FakeSession(
|
||
specs=[stack_error, stack_error, stack_error, HALLUCINATED, HALLUCINATED, HALLUCINATED],
|
||
reviews=[OK, OK, OK, MISMATCH, MISMATCH, MISMATCH],
|
||
codes=[SolverCode(code=STRUCTURED_CODE)],
|
||
)
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=2, max_restarts=1)
|
||
assert result.specification.executive_summary == "richtig, aber Stack falsch"
|
||
assert not result.reliable # STACK_MISMATCH bleibt bestehen
|
||
assert any("STACK_MISMATCH" in issue for issue in result.reliability_issues)
|
||
assert result.solver_run is not None # Zielgröße passt -> Solver läuft, Ergebnis aber als unzuverlässig markiert
|
||
|
||
|
||
def test_no_restart_when_disabled():
|
||
session = FakeSession(specs=[HALLUCINATED] * 3, reviews=[MISMATCH] * 3)
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=2, max_restarts=0)
|
||
assert result.restarts == 0 and not result.reliable
|
||
|
||
|
||
def test_report_shows_prominent_warning_for_unreliable_result():
|
||
from report import build_document, to_html, to_markdown
|
||
session = FakeSession(specs=[HALLUCINATED] * 3, reviews=[MISMATCH] * 3)
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=2, max_restarts=0)
|
||
doc = build_document(result)
|
||
assert doc[1][0] == "warn" # direkt unter der Überschrift
|
||
assert '<div class="warn">⚠ Ergebnis unzuverlässig' in to_html(doc)
|
||
assert "> **⚠ Ergebnis unzuverlässig" in to_markdown(doc)
|
||
assert "Der Solver wurde nicht gestartet." in doc[1][1]
|
||
|
||
|
||
def test_report_shows_the_rejected_reading_of_each_assumption():
|
||
from report import build_document, to_markdown
|
||
spec = make_spec(open_assumptions=[{
|
||
"unclear_point": "„aufeinanderfolgend“",
|
||
"assumption": "Kalendertage, auch über das Wochenende",
|
||
"alternative": "Nur Werktage; verworfen, weil der Text durchgehenden Betrieb nennt",
|
||
"impact": "Restriktion Ruhezeit greift auch am Wochenende",
|
||
}])
|
||
session = FakeSession([spec], [OK], [SolverCode(code=STRUCTURED_CODE)])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, solve_model=False)
|
||
markdown = to_markdown(build_document(result))
|
||
assert "Verworfene Lesart" in markdown
|
||
assert "verworfen, weil der Text durchgehenden Betrieb nennt" in markdown
|
||
|
||
|
||
def test_pipeline_labels_every_llm_call_for_the_timing_breakdown():
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=STRUCTURED_CODE)])
|
||
run_pipeline("Aufgabe", session=session, max_revisions=2, max_restarts=1)
|
||
assert session.steps == ["Verstehen", "Extraktion", "Prüfung", "Solver-Code"]
|
||
|
||
|
||
def test_revision_and_repair_steps_are_labelled():
|
||
session = FakeSession(specs=[HALLUCINATED, GOOD], reviews=[MISMATCH, OK],
|
||
codes=[SolverCode(code="x = 1"), SolverCode(code=STRUCTURED_CODE)])
|
||
run_pipeline("Aufgabe", session=session, max_revisions=1, max_restarts=0)
|
||
assert session.steps == ["Verstehen", "Extraktion", "Prüfung", "Korrektur", "Prüfung", "Solver-Code", "Solver-Reparatur"]
|
||
|
||
|
||
# --- Zulässigkeitsprüfung im Zusammenspiel ---
|
||
|
||
VIOLATING_CODE = '''
|
||
def solve_model():
|
||
"""Liefert eine Lösung, die das Budget um 8 Cent überschreitet (beobachteter Fall)."""
|
||
return 100000.08
|
||
|
||
|
||
def main():
|
||
"""Meldet die Lösung samt Restriktionen zur Nachprüfung."""
|
||
invested = solve_model()
|
||
report("optimal", 12041.74, {"x": invested},
|
||
constraints=[{"name": "Bedarf", "lhs": invested, "sense": "<=", "rhs": 100000.0}],
|
||
variables=[{"name": "x", "value": invested, "var_type": "continuous", "lb": 0}])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|
||
'''
|
||
|
||
|
||
def test_infeasible_solution_triggers_repair_and_is_marked_unreliable():
|
||
session = FakeSession([GOOD], [OK], codes=[SolverCode(code=VIOLATING_CODE)] * 3)
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0, solver_retries=2)
|
||
assert session.calls[SolverCode] == 3 # Erstversuch + 2 Reparaturen
|
||
assert result.solver_run.feasibility and "Bedarf" in result.solver_run.feasibility[0]
|
||
assert not result.reliable
|
||
assert any("Zulässigkeitsprüfung" in issue for issue in result.reliability_issues)
|
||
|
||
|
||
def test_feasible_solution_stays_reliable():
|
||
session = FakeSession([GOOD], [OK], codes=[SolverCode(code=STRUCTURED_CODE)])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0)
|
||
assert result.solver_run.feasibility == []
|
||
assert result.reliable and session.calls[SolverCode] == 1
|
||
|
||
|
||
# --- Gegenprobe bei "unlösbar" ---
|
||
|
||
INFEASIBLE_CODE = '''
|
||
def main():
|
||
"""Meldet das Modell als unlösbar."""
|
||
report("infeasible", None, {})
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|
||
'''
|
||
|
||
|
||
def test_infeasible_is_rechecked_and_can_be_refuted():
|
||
# beobachtet bei qwen3-235b: "unlösbar" war ein Modellierungsfehler, die Aufgabe ist lösbar
|
||
session = FakeSession(specs=[GOOD, GOOD], reviews=[OK, OK],
|
||
codes=[SolverCode(code=INFEASIBLE_CODE), SolverCode(code=STRUCTURED_CODE)])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0)
|
||
assert result.solver_run.status == SolverStatus.OPTIMAL and result.solver_run.objective_value == 9
|
||
assert not result.infeasible_confirmed and result.restarts == 1
|
||
assert "HINWEIS" in session.tasks[1] and "unabhängige Modellierung" in session.tasks[1]
|
||
|
||
|
||
def test_infeasible_confirmed_by_second_modelling():
|
||
session = FakeSession(specs=[GOOD, GOOD], reviews=[OK, OK],
|
||
codes=[SolverCode(code=INFEASIBLE_CODE)] * 2)
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0)
|
||
assert result.solver_run.status == SolverStatus.INFEASIBLE
|
||
assert result.infeasible_confirmed and result.reliable
|
||
|
||
|
||
def test_infeasible_without_recheck_is_flagged(monkeypatch):
|
||
import TextToModel
|
||
monkeypatch.setattr(TextToModel, "INFEASIBLE_RECHECK", 0)
|
||
session = FakeSession([GOOD], [OK], codes=[SolverCode(code=INFEASIBLE_CODE)])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0)
|
||
assert not result.reliable and any("ohne Gegenprobe" in i for i in result.reliability_issues)
|
||
|
||
|
||
# --- Wiedereinstieg ab einer Stufe (continue_pipeline / CLI --von) ---
|
||
|
||
def stored_result(tmp_path, spec=GOOD, review=OK):
|
||
"""Ergebnis eines früheren Laufs als Datei – Grundlage für --von."""
|
||
session = FakeSession([spec], [review], [SolverCode(code=STRUCTURED_CODE)])
|
||
result = run_pipeline("Rohstoffeinkauf", session=session, max_revisions=0, max_restarts=0, solve_model=False)
|
||
path = tmp_path / "ergebnis.json"
|
||
path.write_text(result.model_dump_json(indent=2), encoding="utf-8")
|
||
return path
|
||
|
||
|
||
def test_continue_from_solver_skips_extraction_and_review(tmp_path):
|
||
from TextToModel import continue_pipeline
|
||
path = stored_result(tmp_path)
|
||
session = FakeSession([], [], [SolverCode(code=STRUCTURED_CODE)])
|
||
result = continue_pipeline(str(path), "solver", session=session)
|
||
assert session.steps == ["Solver-Code"] # kein Extraktions-, kein Prüfaufruf
|
||
assert result.specification.executive_summary == GOOD.executive_summary
|
||
assert result.task == "Rohstoffeinkauf" # Aufgabentext kommt aus dem Ergebnis
|
||
assert result.solver_run.status == SolverStatus.OPTIMAL
|
||
|
||
|
||
def test_continue_from_review_keeps_specification_but_checks_again(tmp_path):
|
||
from TextToModel import continue_pipeline
|
||
path = stored_result(tmp_path)
|
||
session = FakeSession([], [OK], [SolverCode(code=STRUCTURED_CODE)])
|
||
result = continue_pipeline(str(path), "pruefung", session=session)
|
||
assert session.steps == ["Prüfung", "Solver-Code"] # Extraktion entfällt
|
||
assert result.specification.executive_summary == GOOD.executive_summary
|
||
|
||
|
||
def test_continue_from_review_can_still_revise(tmp_path):
|
||
from TextToModel import continue_pipeline
|
||
path = stored_result(tmp_path)
|
||
session = FakeSession([GOOD], [MISMATCH, OK], [SolverCode(code=STRUCTURED_CODE)])
|
||
result = continue_pipeline(str(path), "pruefung", session=session, max_revisions=1)
|
||
assert session.steps == ["Prüfung", "Korrektur", "Prüfung", "Solver-Code"]
|
||
assert result.revisions == 1
|
||
|
||
|
||
def test_continue_from_solver_refuses_when_objective_does_not_match(tmp_path):
|
||
# gespeicherte Zielgrößen-Abweichung bleibt bindend: kein Solver, kein LLM-Aufruf
|
||
from TextToModel import continue_pipeline
|
||
path = stored_result(tmp_path, review=MISMATCH)
|
||
session = FakeSession([], [], [])
|
||
result = continue_pipeline(str(path), "solver", session=session)
|
||
assert session.steps == []
|
||
assert result.solver_run is None and not result.reliable
|
||
|
||
|
||
def test_unknown_stage_is_rejected(tmp_path):
|
||
import pytest
|
||
from TextToModel import continue_pipeline
|
||
path = stored_result(tmp_path)
|
||
with pytest.raises(ValueError, match="Unbekannte Stufe"):
|
||
continue_pipeline(str(path), "extraktion", session=FakeSession([], [], []))
|
||
|
||
|
||
def test_warnings_reach_the_revision_prompt():
|
||
# Warnungen loesen keine Runde aus, muessen aber mitkorrigiert werden, wenn ohnehin eine stattfindet
|
||
from model_schema import CheckIssue, CheckSeverity
|
||
from TextToModel import revise_model
|
||
session = FakeSession([GOOD], [], [])
|
||
revise_model(session, "Aufgabe", GOOD, OK, issues=[
|
||
CheckIssue(severity=CheckSeverity.WARNING, code="UNEXPLAINED_JARGON", message="Fachbegriffe: Fluss"),
|
||
CheckIssue(severity=CheckSeverity.ERROR, code="STACK_MISMATCH", message="passt nicht"),
|
||
])
|
||
prompt = session.letzter_prompt
|
||
assert "STACK_MISMATCH" in prompt.split("HINWEISE")[0] # Fehler im Fehlerteil
|
||
assert "UNEXPLAINED_JARGON" in prompt.split("HINWEISE")[1] # Warnung im Hinweisteil
|
||
|
||
|
||
# --- Benchmark haelt fest, womit gemessen wurde ---
|
||
|
||
def test_benchmark_records_the_prompt_state():
|
||
from benchmark import PROMPT_FILES, prompt_state
|
||
stand = prompt_state()
|
||
assert {"prompts", "commit", "dirty"} <= set(stand)
|
||
assert len(stand["prompts"]) == 12 and stand["commit"]
|
||
assert "Routing-und-Extraktions-Prompt.txt" in PROMPT_FILES
|
||
assert "model_schema.py" in PROMPT_FILES # Feldbeschreibungen wirken wie Prompt-Anweisungen
|
||
|
||
|
||
def test_deviating_settings_are_recorded(monkeypatch):
|
||
# Ohne die Umgebung liesse sich ein A/B-Lauf (Stufe abgeschaltet, anderes Modell) nicht zuordnen
|
||
import benchmark
|
||
benchmark.prompt_state.cache_clear()
|
||
monkeypatch.setenv("UNDERSTAND_RETRIES", "-1")
|
||
monkeypatch.setenv("LLM_MODEL_SOLVER", "billig/modell")
|
||
stand = benchmark.prompt_state()
|
||
assert stand["config"] == {"UNDERSTAND_RETRIES": "-1", "LLM_MODEL_SOLVER": "billig/modell"}
|
||
benchmark.prompt_state.cache_clear()
|
||
|
||
|
||
def test_prompt_digest_changes_with_the_prompt(tmp_path):
|
||
from benchmark import prompt_digest
|
||
a, b = tmp_path / "eins.txt", tmp_path / "zwei.txt"
|
||
a.write_text("Regel A"), b.write_text("Regel B")
|
||
vorher = prompt_digest([a, b])
|
||
assert prompt_digest([a, b]) == vorher # gleicher Inhalt, gleiche Kennung
|
||
b.write_text("Regel B, nun schaerfer")
|
||
assert prompt_digest([a, b]) != vorher # eine geaenderte Regel aendert die Kennung
|
||
assert prompt_digest([b, a]) != prompt_digest([a, b]) # Reihenfolge ist Teil der Kennung
|
||
|
||
|
||
def test_unexplained_jargon_triggers_a_revision_but_stays_reliable():
|
||
# Erklaeren ist Kernziel: die Warnung ist eine Runde wert, macht das Ergebnis aber nicht unzuverlaessig
|
||
from model_checks import check_spec
|
||
unerklaert = make_spec(
|
||
executive_summary="Der Betrieb entscheidet ueber Mengen.",
|
||
constraints=[{"identifier": "Flusserhaltung", "latex_formula": "\\sum_j x_j = 1", "description": "-"}])
|
||
assert "UNEXPLAINED_JARGON" in {i.code for i in check_spec(unerklaert)}
|
||
|
||
session = FakeSession(specs=[unerklaert, GOOD], reviews=[OK, OK], codes=[SolverCode(code=STRUCTURED_CODE)])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=1, max_restarts=0)
|
||
assert session.steps == ["Verstehen", "Extraktion", "Prüfung", "Korrektur", "Prüfung", "Solver-Code"]
|
||
assert result.revisions == 1
|
||
assert result.reliable # Warnung, kein Fehler
|
||
|
||
|
||
def test_clean_model_still_needs_no_revision():
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=STRUCTURED_CODE)])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=2, max_restarts=0)
|
||
assert result.revisions == 0
|
||
|
||
|
||
# --- Verstehensstufe im Ablauf ---
|
||
|
||
def test_understanding_runs_first_and_is_kept_in_the_result():
|
||
from model_schema import Fact
|
||
verstanden = TaskUnderstanding(
|
||
is_optimization_task=True, reason="Kostenminimierung", ignored_numbers=[],
|
||
sub_questions=[SubQuestion(id="a", quote="Wie viel?", kind="rechnen", needs_solver=True)],
|
||
facts=[Fact(value="12", unit="€", meaning="Deckungsbeitrag je Stueck", quote="12 € je Stueck")])
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=STRUCTURED_CODE)], understandings=[verstanden])
|
||
result = run_pipeline("Der Deckungsbeitrag ist 12 € je Stueck.", session=session,
|
||
max_revisions=0, max_restarts=0)
|
||
assert session.steps[0] == "Verstehen"
|
||
assert result.understanding.facts[0].value == "12"
|
||
assert [q.id for q in result.understanding.sub_questions] == ["a"]
|
||
|
||
|
||
def test_incomplete_understanding_is_asked_again():
|
||
# erste Antwort uebersieht die Zahl, zweite erfasst sie
|
||
from model_schema import Fact
|
||
luecke = TaskUnderstanding(is_optimization_task=True, reason="-", facts=[], ignored_numbers=[],
|
||
sub_questions=[SubQuestion(id="1", quote="-", kind="rechnen", needs_solver=True)])
|
||
vollstaendig = luecke.model_copy(update={
|
||
"facts": [Fact(value="12", unit="€", meaning="Deckungsbeitrag", quote="12 € je Stueck")]})
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=STRUCTURED_CODE)],
|
||
understandings=[luecke, vollstaendig])
|
||
result = run_pipeline("12 € je Stueck.", session=session, max_revisions=0, max_restarts=0)
|
||
assert session.steps[:2] == ["Verstehen", "Verstehen"] # genau ein Nachfassen
|
||
assert result.understanding.facts[0].value == "12"
|
||
|
||
|
||
def test_report_shows_questions_and_numbers():
|
||
from model_schema import Fact
|
||
from report import build_document, to_markdown
|
||
verstanden = TaskUnderstanding(
|
||
is_optimization_task=True, reason="-",
|
||
sub_questions=[SubQuestion(id="a", quote="Formulieren Sie das Problem!", kind="formulieren", needs_solver=False),
|
||
SubQuestion(id="b", quote="Loesen Sie es!", kind="rechnen", needs_solver=True)],
|
||
facts=[Fact(value="8", unit="€", meaning="Kosten im ersten Jahr", quote="Zeile 0: 8 | 18 | 31")],
|
||
ignored_numbers=[])
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=STRUCTURED_CODE)], understandings=[verstanden])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0)
|
||
markdown = to_markdown(build_document(result))
|
||
assert "Gestellte Fragen" in markdown and "Formulieren Sie das Problem!" in markdown
|
||
assert "Gegebene Zahlen" in markdown and "Kosten im ersten Jahr" in markdown
|
||
|
||
|
||
def test_report_puts_the_answers_before_the_numbers():
|
||
from report import build_document, to_markdown
|
||
code = '''
|
||
def main():
|
||
"""Loest und beantwortet."""
|
||
report("optimal", 9.0, {"x": 1.0},
|
||
headline="Die Kosten betragen 9 Geldeinheiten.",
|
||
constraints=[{"name": "Bedarf", "lhs": 4.0, "sense": ">=", "rhs": 4.0}],
|
||
variables=[{"name": "x", "value": 1.0, "var_type": "continuous", "lb": 0}],
|
||
answers=[{"question_id": "a", "text": "Das Modell ist ein lineares Programm."},
|
||
{"question_id": "b", "text": "Die minimalen Kosten betragen 9 Geldeinheiten."}])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|
||
'''
|
||
verstanden = TaskUnderstanding(
|
||
is_optimization_task=True, reason="-", facts=[], ignored_numbers=[],
|
||
sub_questions=[SubQuestion(id="a", quote="Formulieren Sie das Modell!", kind="formulieren", needs_solver=False),
|
||
SubQuestion(id="b", quote="Berechnen Sie die Kosten!", kind="rechnen", needs_solver=True)])
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=code)], understandings=[verstanden])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0)
|
||
markdown = to_markdown(build_document(result))
|
||
assert "Antworten auf die Aufgabe" in markdown
|
||
assert markdown.index("Formulieren Sie das Modell!") < markdown.index("| Status |")
|
||
assert result.solver_run.warnings == [] # Abnahme ohne Befund
|
||
|
||
|
||
def test_unanswered_sub_question_is_a_warning_not_a_wrong_number():
|
||
code = '''
|
||
def main():
|
||
"""Beantwortet nur die zweite Frage."""
|
||
report("optimal", 9.0, {"x": 1.0},
|
||
constraints=[{"name": "Bedarf", "lhs": 4.0, "sense": ">=", "rhs": 4.0}],
|
||
variables=[{"name": "x", "value": 1.0, "var_type": "continuous", "lb": 0}],
|
||
answers=[{"question_id": "b", "text": "Die minimalen Kosten betragen 9 Geldeinheiten."}])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|
||
'''
|
||
verstanden = TaskUnderstanding(
|
||
is_optimization_task=True, reason="-", facts=[], ignored_numbers=[],
|
||
sub_questions=[SubQuestion(id="a", quote="Zeichnen Sie den Graphen!", kind="zeichnen", needs_solver=False),
|
||
SubQuestion(id="b", quote="Berechnen Sie die Kosten!", kind="rechnen", needs_solver=True)])
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=code)] * 3, understandings=[verstanden])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0)
|
||
assert any("Teilfragen ohne Antwort" in w for w in result.solver_run.warnings)
|
||
assert result.solver_run.objective_value == 9.0 # die Zahl bleibt richtig
|
||
|
||
|
||
def test_report_shows_data_profile_and_scenarios(tmp_path):
|
||
from model_schema import Scenario
|
||
from report import build_document, to_markdown
|
||
datei = tmp_path / "werte.csv"
|
||
datei.write_text("name,menge\nA,3\nB,4\n")
|
||
code = '''
|
||
DATA = {"faktor": 1.0}
|
||
|
||
|
||
def main():
|
||
"""Meldet zwei Datenvarianten."""
|
||
report("optimal", 7.0, {"x": 1.0},
|
||
constraints=[{"name": "Bedarf", "lhs": 4.0, "sense": ">=", "rhs": 4.0}],
|
||
variables=[{"name": "x", "value": 1.0, "var_type": "continuous", "lb": 0}],
|
||
scenarios=[{"name": "Doppelt", "description": "Faktor verdoppelt",
|
||
"data_override": {"faktor": 2.0}, "objective_value": 7.0}])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|
||
'''
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=code)])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0, data_files=[datei])
|
||
markdown = to_markdown(build_document(result))
|
||
assert "Beigelegte Daten" in markdown and "werte.csv (2 Zeilen)" in markdown
|
||
assert "Verglichene Konfigurationen" in markdown and "Doppelt" in markdown
|
||
assert result.data_files[0].rows == 2
|
||
assert result.solver_run.scenarios[0].verified_value == 7.0 # ohne LLM nachgerechnet
|
||
|
||
|
||
# --- Benchmark misst "Aufgabe erfuellt", nicht nur "Zahl richtig" ---
|
||
|
||
def test_benchmark_counts_unanswered_questions_as_not_fulfilled(monkeypatch):
|
||
import benchmark
|
||
from model_schema import Answer, SubQuestion, TaskUnderstanding
|
||
|
||
def ergebnis(antworten, warnungen=()):
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=STRUCTURED_CODE)], understandings=[TaskUnderstanding(
|
||
is_optimization_task=True, reason="-", facts=[], ignored_numbers=[],
|
||
sub_questions=[SubQuestion(id=f, quote="-", kind="rechnen", needs_solver=True) for f in ("a", "b")])])
|
||
r = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0)
|
||
r.solver_run.answers = [Answer(question_id=k, text="-") for k in antworten]
|
||
r.solver_run.warnings = list(warnungen)
|
||
return r
|
||
|
||
fall = {"name": "test", "task": "Aufgabe", "expected": {"objective_value": 9, "tolerance": 1e-6}}
|
||
for antworten, warnungen, erwartet in [
|
||
(["a", "b"], [], True), # alles beantwortet
|
||
(["b"], ["Teilfragen ohne Antwort: a"], False), # eine Frage offen
|
||
(["a", "b"], ["Antwort zu 'b' nennt Zahlen …"], False), # unbelegte Zahl
|
||
]:
|
||
monkeypatch.setattr(benchmark, "run_pipeline", lambda *a, **k: ergebnis(antworten, warnungen))
|
||
zeile = benchmark.run_case(fall, "fake", 1)
|
||
assert zeile["objective_ok"] is True # die Zahl stimmt immer
|
||
assert zeile["task_ok"] is erwartet, (antworten, warnungen)
|
||
|
||
|
||
def test_benchmark_case_data_files_are_collected(tmp_path, monkeypatch):
|
||
import benchmark
|
||
fall = tmp_path / "cases" / "mit_daten"
|
||
fall.mkdir(parents=True)
|
||
(fall / "task.txt").write_text("Aufgabe")
|
||
(fall / "expected.json").write_text('{"objective_value": 1, "tolerance": 1e-6}')
|
||
(fall / "werte.csv").write_text("a,b\n1,2\n")
|
||
monkeypatch.setattr(benchmark, "CASES_DIR", tmp_path / "cases")
|
||
faelle = benchmark.load_cases()
|
||
assert [p.split("/")[-1] for p in faelle[0]["data_files"]] == ["werte.csv"]
|
||
|
||
|
||
# Spezifikation ohne Kostendaten, Programm erfindet sie per Zufallsgenerator (Muster "Portfolio Nr. 2")
|
||
OHNE_DATEN = make_spec(parameters=[
|
||
{"symbol": "c_j", "dimension": "Vektor", "unit": "€/t", "description": "Kosten", "extracted_values": ""},
|
||
{"symbol": "b", "dimension": "Skalar", "unit": "t", "description": "Bedarf", "extracted_values": "4"}])
|
||
ERFUNDENE_DATEN = STRUCTURED_CODE.replace(
|
||
"COSTS = [3, 2]", "import numpy as np\nCOSTS = [float(v) for v in np.random.default_rng(0).integers(2, 4, 2)]")
|
||
|
||
|
||
def test_undeclared_generated_data_is_repaired():
|
||
session = FakeSession([OHNE_DATEN], [OK], [
|
||
SolverCode(code=ERFUNDENE_DATEN),
|
||
SolverCode(code=ERFUNDENE_DATEN, data_assumptions=["c_j: zufällig erzeugt (Seed 0), keine Daten im Text"])])
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0, solver_retries=1)
|
||
assert session.calls[SolverCode] == 2 # der Befund hat eine Reparatur ausgelöst
|
||
assert result.reliable and result.solver_run.undeclared_data == []
|
||
|
||
|
||
def test_persistently_undeclared_data_is_unreliable():
|
||
# Nachmessung "Portfolio Nr. 2": data_assumptions leer, Antwort behauptete geladene Kursdaten
|
||
session = FakeSession([OHNE_DATEN], [OK], [SolverCode(code=ERFUNDENE_DATEN)] * 2)
|
||
result = run_pipeline("Aufgabe", session=session, max_revisions=0, max_restarts=0, solver_retries=1)
|
||
assert not result.reliable
|
||
assert any(p.startswith("Datenherkunft") for p in result.reliability_issues)
|