fix: abgeschnittene Antworten meiden den Anbieter; LLM-Ausfall macht unzuverlässig statt abzustürzen
Grundlinie 20260918-191400: 9 von 22 Läufen hatten Aufrufe, die an LLM_MAX_TOKENS abgeschnitten wurden (Venice 4/28, Makora 2/43, übrige 0/54); hard_portfolio_kardinalitaet brach nach vier Abschneidungen ab, Kosten gingen verloren. - Abschneiden: sofort neu versuchen, Anbieter per provider.ignore meiden (_avoid_provider, gemeinsam mit „zu langsam“) - nach allen Wiederholungen LLMUnavailable (Stufe + Ursache); fatale Fehler bleiben unverpackt - Solver-Stufe behält Modell und Prüfbericht (Solver-Fehler), Gegenprobe behält erstes Ergebnis, davor Ergebnis ohne Spezifikation mit Kosten („LLM-Fehler“, Report „Lauf abgebrochen“) - Benchmark: Status „LLM-Fehler“ statt ABBRUCH, zählt nicht als ehrliches Scheitern - TextToModel.py: Shebang Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
422360ffa5
commit
6b2b2c6466
1 changed files with 218 additions and 25 deletions
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
|
|
@ -8,8 +10,9 @@ from typing import List
|
|||
from checks.model_checks import check_figure_readings, check_spec, check_understanding
|
||||
from checks.review import needs_work, review_assumptions, revision_reasons, select_best
|
||||
from core import llm
|
||||
from core.llm import DEFAULT_MODEL, LLMSession
|
||||
from core.llm import DEFAULT_MODEL, LLMSession, LLMUnavailable
|
||||
from core.model_schema import (
|
||||
SolverRun,
|
||||
SolverStatus,
|
||||
CheckSeverity,
|
||||
OptimizationModelSpecification,
|
||||
|
|
@ -157,6 +160,18 @@ def _recheck_infeasible(session, task: str, model: str, result: PipelineResult,
|
|||
return result
|
||||
|
||||
log(f"[{model}] Solver meldet '{status}' – Gegenprobe mit unabhängiger Modellierung ...")
|
||||
try:
|
||||
return _run_recheck(session, task, model, result, status, max_revisions, solver_retries,
|
||||
data_files, data_prompt)
|
||||
except LLMUnavailable as exc:
|
||||
log(f" Gegenprobe abgebrochen: {exc}")
|
||||
result.reliable = False
|
||||
result.reliability_issues.append(f"Solver meldet '{status}'; die Gegenprobe scheiterte am LLM ({exc.step})")
|
||||
return result
|
||||
|
||||
|
||||
def _run_recheck(session, task: str, model: str, result: PipelineResult, status: str, max_revisions: int,
|
||||
solver_retries: int, data_files, data_prompt: str) -> PipelineResult:
|
||||
zweiter = _model_attempt(session, task, max_revisions, f"{model} Gegenprobe",
|
||||
hint=INFEASIBLE_HINT.format(status=status),
|
||||
understanding=result.understanding, data_prompt=data_prompt)
|
||||
|
|
@ -201,9 +216,17 @@ def _solver_stage(session, task: str, result: PipelineResult, model: str,
|
|||
return result
|
||||
|
||||
log(f"[{model}] Erzeuge und starte Solver-Code ({spec.target_stack.value}) ...")
|
||||
result.solver_code, result.solver_run = solve(session, task, spec, retries=solver_retries, log=log,
|
||||
understanding=result.understanding,
|
||||
data_files=data_files, data_prompt=data_prompt)
|
||||
try:
|
||||
result.solver_code, result.solver_run = solve(session, task, spec, retries=solver_retries, log=log,
|
||||
understanding=result.understanding,
|
||||
data_files=data_files, data_prompt=data_prompt)
|
||||
except LLMUnavailable as exc:
|
||||
# Modell und Prüfbericht bleiben erhalten – `--von solver` kann genau hier weitermachen
|
||||
result.solver_run = SolverRun(status=SolverStatus.ERROR, error=str(exc))
|
||||
result.reliable = False
|
||||
result.reliability_issues.append(f"Solver-Fehler: {exc}")
|
||||
log(f"ERGEBNIS UNZUVERLÄSSIG – {exc}")
|
||||
return result
|
||||
_assess_run(result)
|
||||
if result.solver_run.status in (SolverStatus.INFEASIBLE, SolverStatus.UNBOUNDED):
|
||||
result = _recheck_infeasible(session, task, model, result, max_revisions, solver_retries,
|
||||
|
|
@ -270,6 +293,20 @@ def run_pipeline(
|
|||
session = session or LLMSession(model)
|
||||
# Abbildungen (docling: Base64 oder Bilddatei) gehen nur an die Verstehensstufe; ab dann zählt ihre Übertragung
|
||||
task, figures = split_figures(task, task_dir)
|
||||
try:
|
||||
return _run_pipeline(session, task, figures, model, max_revisions, solve_model, solver_retries,
|
||||
max_restarts, data_files)
|
||||
except LLMUnavailable as exc:
|
||||
# Vor der Solver-Stufe gibt es nichts Verwertbares; statt Absturz ein ehrliches, bezahltes Nicht-Ergebnis
|
||||
log(f"ERGEBNIS UNZUVERLÄSSIG – {exc}")
|
||||
result = PipelineResult(task=task, figures=figures, reliable=False,
|
||||
reliability_issues=[f"LLM-Fehler: {exc}"])
|
||||
_log_usage(session, result)
|
||||
return result
|
||||
|
||||
|
||||
def _run_pipeline(session, task: str, figures, model: str, max_revisions: int, solve_model: bool,
|
||||
solver_retries: int, max_restarts: int, data_files) -> PipelineResult:
|
||||
if figures:
|
||||
log(f"{len(figures)} Abbildung(en) in der Aufgabe – werden in der Verstehensstufe in Text übertragen.")
|
||||
profile = data_profile.profile_files(data_files)
|
||||
|
|
@ -367,8 +404,15 @@ def continue_pipeline(
|
|||
problems = _unresolved({"review": stored.review, "issues": checks})
|
||||
update = {"checks": checks}
|
||||
else:
|
||||
attempt = _model_attempt(session, task, max_revisions, model, spec=stored.specification,
|
||||
understanding=understanding, data_prompt=data_prompt)
|
||||
try:
|
||||
attempt = _model_attempt(session, task, max_revisions, model, spec=stored.specification,
|
||||
understanding=understanding, data_prompt=data_prompt)
|
||||
except LLMUnavailable as exc:
|
||||
log(f"ERGEBNIS UNZUVERLÄSSIG – {exc}")
|
||||
result = stored.model_copy(update={"solver_code": None, "solver_run": None, "reliable": False,
|
||||
"reliability_issues": [f"LLM-Fehler: {exc}"]})
|
||||
_log_usage(session, result)
|
||||
return result
|
||||
problems = _unresolved(attempt)
|
||||
update = {"specification": attempt["spec"], "review": attempt["review"], "revisions": attempt["revisions"],
|
||||
"selected_revision": attempt["selected"], "restarts": 0, "checks": attempt["issues"]}
|
||||
|
|
@ -590,7 +634,9 @@ def summary(result: PipelineResult) -> str:
|
|||
return "Abgelehnt – " + "; ".join(result.reliability_issues)
|
||||
zeilen = []
|
||||
run = result.solver_run
|
||||
if run is None:
|
||||
if result.specification is None:
|
||||
zeilen.append("Kein Modell entstanden.")
|
||||
elif run is None:
|
||||
spec = result.specification
|
||||
zeilen.append(f"Modell: {spec.problem_class.value} → {spec.target_stack.value} (kein Solver-Lauf)")
|
||||
else:
|
||||
|
|
|
|||
Loading…
Reference in a new issue