fix: antwortendes Modell erfassen; 0.040, LaTeX-Dezimalkomma und Teilfragen-Überschriften
- CallRecord.responded_model / LLMUsage.responded_models: tatsächlich antwortendes
Modell (beim Alias ~deepseek/deepseek-flash-latest) in Log, Report und Benchmark
- normalize_number: führende Null ist nie Tausender (0.040 wurde als 40 gelesen)
- numbers_in: LaTeX-Dezimalkomma 0{,}08 als eine Zahl
- question_ids_in: '### Teil N' als Teilfrage; ausdrückliche Marker haben Vorrang
vor nummerierten Listen
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
88adcff586
commit
8bd3c2e7e1
1 changed files with 88 additions and 10 deletions
|
|
@ -125,6 +125,7 @@ def run_case(case: dict, model: str, run_no: int) -> dict:
|
|||
objective_value=None, objective_ok=False, questions=fragen, answered=0, answer_findings=0,
|
||||
task_ok=False, solver_error="; ".join(result.reliability_issues), reliable=False, restarts=0,
|
||||
providers=result.usage.providers if result.usage else {},
|
||||
responded_models=result.usage.responded_models if result.usage else {},
|
||||
llm_calls=result.usage.calls if result.usage else 0,
|
||||
reasoning_tokens=result.usage.reasoning_tokens if result.usage else 0,
|
||||
cost_usd=result.usage.cost_usd if result.usage else None,
|
||||
|
|
@ -159,6 +160,7 @@ def run_case(case: dict, model: str, run_no: int) -> dict:
|
|||
reliable=result.reliable,
|
||||
restarts=result.restarts,
|
||||
providers=result.usage.providers,
|
||||
responded_models=result.usage.responded_models,
|
||||
llm_calls=result.usage.calls,
|
||||
reasoning_tokens=result.usage.reasoning_tokens,
|
||||
cost_usd=result.usage.cost_usd,
|
||||
|
|
@ -208,6 +210,12 @@ def print_table(rows):
|
|||
cost = sum(r.get("cost_usd") or 0 for r in mine)
|
||||
print(f" {model}: Ergebnis korrekt {hits}/{len(mine)}, Aufgabe erfüllt {erfuellt}/{len(mine)}, Routing korrekt {routing}/{len(mine)}, "
|
||||
f"Ø {seconds:.0f} s pro Lauf, Kosten gesamt ${cost:.4f} (ohne abgebrochene Läufe)")
|
||||
antwortend = {}
|
||||
for r in mine:
|
||||
for name, n in (r.get("responded_models") or {}).items():
|
||||
antwortend[name] = antwortend.get(name, 0) + n
|
||||
if antwortend and set(antwortend) != {model}:
|
||||
print(f" Modell laut Antwort: {antwortend}")
|
||||
|
||||
for r in rows:
|
||||
if r.get("error") or r.get("solver_error"):
|
||||
|
|
|
|||
Loading…
Reference in a new issue