dschlueter
b74445b8d6
Übersetzt natürlichsprachige Optimierungsaufgaben (Deutsch) per LLM in eine geprüfte Modellspezifikation, erzeugt Solver-Code und führt ihn in einer bubblewrap-Sandbox aus. Deterministische Prüfungen ohne LLM, Report als Markdown/HTML, Benchmark mit 19 verifizierten Fällen. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
66 lines
2.6 KiB
Python
66 lines
2.6 KiB
Python
from model_schema import AssumptionReview, CheckIssue, CheckSeverity
|
|
from TextToModel import select_best
|
|
|
|
|
|
def make_review(matches=True, verdict="justified", missing=(), units_consistent=True):
|
|
return AssumptionReview.model_validate({
|
|
"objective_check": {
|
|
"task_evidence": "Bestimme die Aufteilung mit der geringsten Volatilität",
|
|
"asked_quantity": "Standardabweichung der Portfoliorendite",
|
|
"modeled_quantity": "Standardabweichung" if matches else "Varianz",
|
|
"matches": matches,
|
|
"units_consistent": units_consistent,
|
|
"unit_explanation": "Rendite €, Risiko €, Kosten €" if units_consistent else "Rendite €, Risiko dimensionslos, Kosten €",
|
|
"explanation": "-",
|
|
},
|
|
"checks": [{"unclear_point": "-", "verdict": verdict, "task_evidence": "-", "explanation": "-"}],
|
|
"missing_assumptions": list(missing),
|
|
})
|
|
|
|
|
|
def test_clean_review_needs_no_revision():
|
|
assert not make_review().needs_revision
|
|
|
|
|
|
def test_objective_mismatch_triggers_revision():
|
|
assert make_review(matches=False).needs_revision
|
|
|
|
|
|
def test_faulty_assumption_triggers_revision():
|
|
assert make_review(verdict="stricter_than_task").needs_revision
|
|
|
|
|
|
def test_missing_assumption_triggers_revision():
|
|
missing = [{"unclear_point": "-", "assumption": "-", "alternative": "-", "impact": "-"}]
|
|
assert make_review(missing=missing).needs_revision
|
|
|
|
|
|
# --- Auswahl der besten Korrekturrunde (TextToModel.select_best) ---
|
|
|
|
ERROR = CheckIssue(severity=CheckSeverity.ERROR, code="CPSAT_WITH_CONTINUOUS_VARS", message="-")
|
|
WARNING = CheckIssue(severity=CheckSeverity.WARNING, code="UNDECLARED_SYMBOL", message="-")
|
|
|
|
|
|
def test_later_hallucinated_round_is_discarded():
|
|
# beobachtet in portfolio_qp: Runde 1 sauber, Runde 2 beschreibt ein Fremdproblem mit Check-Error
|
|
candidates = [
|
|
(None, [], make_review(matches=False)),
|
|
(None, [WARNING], make_review()),
|
|
(None, [ERROR, WARNING], make_review(matches=False)),
|
|
]
|
|
assert select_best(candidates) == 1
|
|
|
|
|
|
def test_clean_later_round_wins_tie():
|
|
candidates = [(None, [], make_review()), (None, [WARNING], make_review())]
|
|
assert select_best(candidates) == 1
|
|
|
|
|
|
def test_fewer_errors_preferred_when_all_have_errors():
|
|
candidates = [(None, [ERROR], make_review(matches=False)), (None, [ERROR, ERROR], make_review(matches=False))]
|
|
assert select_best(candidates) == 0
|
|
|
|
|
|
def test_mixed_units_trigger_revision():
|
|
# beobachtet: Rendite/Kosten in Euro, Risiko als dimensionslose Summe w_i^2 sigma_i^2 (Faktor 100.000 zu klein)
|
|
assert make_review(units_consistent=False).needs_revision
|