diff --git a/TextToModel.py b/TextToModel.py index 151d7c9..53dddac 100644 --- a/TextToModel.py +++ b/TextToModel.py @@ -229,6 +229,8 @@ def _log_usage(session, result: PipelineResult) -> None: result.usage = session.usage cost = f", Kosten ${session.usage.cost_usd:.4f}" if session.usage.cost_usd is not None else "" providers = f", Anbieter: {session.usage.providers}" if session.usage.providers else "" + if session.usage.responded_models: + providers += f", Modell laut Antwort: {session.usage.responded_models}" usage = session.usage cache = (f" (davon {usage.cached_tokens} aus dem Cache, {100 * usage.cached_tokens / usage.prompt_tokens:.0f} %)" if usage.prompt_tokens else "") diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index f70b1c1..7b011b8 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -125,6 +125,7 @@ def run_case(case: dict, model: str, run_no: int) -> dict: objective_value=None, objective_ok=False, questions=fragen, answered=0, answer_findings=0, task_ok=False, solver_error="; ".join(result.reliability_issues), reliable=False, restarts=0, providers=result.usage.providers if result.usage else {}, + responded_models=result.usage.responded_models if result.usage else {}, llm_calls=result.usage.calls if result.usage else 0, reasoning_tokens=result.usage.reasoning_tokens if result.usage else 0, cost_usd=result.usage.cost_usd if result.usage else None, @@ -159,6 +160,7 @@ def run_case(case: dict, model: str, run_no: int) -> dict: reliable=result.reliable, restarts=result.restarts, providers=result.usage.providers, + responded_models=result.usage.responded_models, llm_calls=result.usage.calls, reasoning_tokens=result.usage.reasoning_tokens, cost_usd=result.usage.cost_usd, @@ -208,6 +210,12 @@ def print_table(rows): cost = sum(r.get("cost_usd") or 0 for r in mine) print(f" {model}: Ergebnis korrekt {hits}/{len(mine)}, Aufgabe erfüllt {erfuellt}/{len(mine)}, Routing korrekt {routing}/{len(mine)}, " f"Ø {seconds:.0f} s pro Lauf, Kosten gesamt ${cost:.4f} (ohne abgebrochene Läufe)") + antwortend = {} + for r in mine: + for name, n in (r.get("responded_models") or {}).items(): + antwortend[name] = antwortend.get(name, 0) + n + if antwortend and set(antwortend) != {model}: + print(f" Modell laut Antwort: {antwortend}") for r in rows: if r.get("error") or r.get("solver_error"): diff --git a/checks/model_checks.py b/checks/model_checks.py index 881e8e7..9a849ee 100644 --- a/checks/model_checks.py +++ b/checks/model_checks.py @@ -113,7 +113,7 @@ NUMBER_PATTERN = re.compile(r"(?]*\**\s*(?:\(?([a-h])\)|(?:Teil|Aufgabe|Frage|Part|Question|Task)\s+(\d+)|([1-9])\.)\s*[:.)]?\**\s+\S", +QUESTION_PATTERN = re.compile(r"^[\s\-*>]*\**\s*(?:\(?([a-h])\)|(?:#{1,6}\s*)?(?:Teil|Aufgabe|Frage|Part|Question|Task)\s+(\d+)|([1-9])\.)\s*[:.)]?\**\s+\S", re.MULTILINE) @@ -136,7 +136,8 @@ def normalize_number(roh: str) -> str: roh = roh.replace(".", "").replace(",", ".") elif "." in roh: teile = roh.split(".") - if all(len(t) == 3 for t in teile[1:]) and len(teile[0]) <= 3: + # „0.040“ ist nie eine Tausenderschreibweise – vorher wurde es zu 40 (portfolio_qp, Rückrechnung 17.09.2026) + if all(len(t) == 3 for t in teile[1:]) and len(teile[0]) <= 3 and teile[0] != "0": roh = "".join(teile) # 1.234 bzw. 12.345.678 wert = float(roh) return str(int(wert)) if wert.is_integer() else f"{wert:g}" # große Zahlen exakt, nicht 1.23457e+06 @@ -145,7 +146,9 @@ def normalize_number(roh: str) -> str: def numbers_in(text: str) -> Set[str]: """Zahlen eines Textes in einheitlicher Schreibweise.""" gefunden = set() - for roh in NUMBER_PATTERN.findall(text or ""): + # LaTeX schreibt das Dezimalkomma als {,} (0{,}08) – sonst zerfiele die Zahl in 0 und 08 + text = re.sub(r"(?<=\d)\{,\}(?=\d)", ",", text or "") + for roh in NUMBER_PATTERN.findall(text): try: gefunden.add(normalize_number(roh)) except ValueError: @@ -154,8 +157,15 @@ def numbers_in(text: str) -> Set[str]: def question_ids_in(text: str) -> Set[str]: - """Kennungen der Teilfragen, die im Aufgabentext erkennbar sind.""" - return {(a or b or c).lower() for a, b, c in QUESTION_PATTERN.findall(text or "")} + """Kennungen der Teilfragen, die im Aufgabentext erkennbar sind. + + Nennt der Text Teilfragen ausdrücklich („### Teil 3“, „Aufgabe 2“), zählen bloße Nummerierungen („1.“) nicht – + beobachtet: eine nummerierte Liste von Restriktionen galt als acht Teilfragen, die als Überschriften gesetzten + eigentlichen Teilaufgaben wurden übersehen. + """ + treffer = QUESTION_PATTERN.findall(text or "") + ausdruecklich = any(b for _, b, _ in treffer) + return {(a or b or c).lower() for a, b, c in treffer if not (ausdruecklich and c and not (a or b))} def _covers(genannt: str, erkannt: str) -> bool: diff --git a/core/llm.py b/core/llm.py index b80e33c..c5ed520 100644 --- a/core/llm.py +++ b/core/llm.py @@ -146,6 +146,13 @@ class LLMSession: provider = getattr(completion, "provider", None) # OpenRouter: tatsächlich antwortender Anbieter if provider: self.usage.providers[provider] = self.usage.providers.get(provider, 0) + 1 + # Beim Alias (~deepseek/deepseek-flash-latest) steht erst in der Antwort, welches Modell gerechnet hat; + # ohne diese Angabe wären Läufe vor und nach einem Modellwechsel des Alias nicht zu unterscheiden. + antwortend = getattr(completion, "model", None) or getattr(completion, "model_version", None) + if not isinstance(antwortend, str): + antwortend = None + if antwortend: + self.usage.responded_models[antwortend] = self.usage.responded_models.get(antwortend, 0) + 1 usage = getattr(completion, "usage", None) if usage is not None: # OpenAI-Format (OpenRouter) self.usage.prompt_tokens += usage.prompt_tokens or 0 @@ -157,7 +164,7 @@ class LLMSession: cost = getattr(usage, "cost", None) if cost is not None: self.usage.cost_usd = (self.usage.cost_usd or 0.0) + cost - self._log_call(step, seconds, before, getattr(completion, "provider", None), model) + self._log_call(step, seconds, before, getattr(completion, "provider", None), model, antwortend) return meta = getattr(completion, "usage_metadata", None) if meta is not None: # Gemini-Format, keine Kostenangabe @@ -165,11 +172,11 @@ class LLMSession: self.usage.completion_tokens += meta.candidates_token_count or 0 self.usage.reasoning_tokens += meta.thoughts_token_count or 0 self.usage.cached_tokens += getattr(meta, "cached_content_token_count", 0) or 0 - self._log_call(step, seconds, before, getattr(completion, "provider", None), model) + self._log_call(step, seconds, before, getattr(completion, "provider", None), model, antwortend) - def _log_call(self, step, seconds, before, provider, model=None) -> None: + def _log_call(self, step, seconds, before, provider, model=None, responded_model=None) -> None: self.usage.call_records.append(CallRecord( - step=step, seconds=round(seconds, 1), provider=provider, model=model, + step=step, seconds=round(seconds, 1), provider=provider, model=model, responded_model=responded_model, completion_tokens=self.usage.completion_tokens - before[0], reasoning_tokens=self.usage.reasoning_tokens - before[1], prompt_tokens=self.usage.prompt_tokens - before[2], diff --git a/core/model_schema.py b/core/model_schema.py index 5ecf0d9..bbdd925 100644 --- a/core/model_schema.py +++ b/core/model_schema.py @@ -427,7 +427,9 @@ class CallRecord(BaseModel): prompt_tokens: int = 0 cached_tokens: int = Field(default=0, description="Input-Tokens aus dem Anbieter-Cache (billiger)") provider: Optional[str] = None - model: Optional[str] = Field(default=None, description="Tatsächlich benutztes Modell (kann je Stufe abweichen)") + model: Optional[str] = Field(default=None, description="Angefragtes Modell (kann je Stufe abweichen)") + responded_model: Optional[str] = Field( + default=None, description="Modell laut Antwort – bei Aliasen wie ~deepseek/deepseek-flash-latest das tatsächliche") class LLMUsage(BaseModel): @@ -440,6 +442,8 @@ class LLMUsage(BaseModel): cached_tokens: int = Field(default=0, description="davon Input-Tokens aus dem Anbieter-Cache") cost_usd: Optional[float] = Field(default=None, description="nur OpenRouter liefert Kosten") providers: Dict[str, int] = Field(default_factory=dict, description="Antwortender Anbieter -> Anzahl Aufrufe (OpenRouter)") + responded_models: Dict[str, int] = Field( + default_factory=dict, description="Tatsächlich antwortendes Modell -> Anzahl Aufrufe (bei Aliasen vergleichsentscheidend)") call_records: List[CallRecord] = Field(default_factory=list) def by_step(self) -> Dict[str, dict]: diff --git a/reporting/report.py b/reporting/report.py index 912d577..eeed453 100644 --- a/reporting/report.py +++ b/reporting/report.py @@ -359,6 +359,8 @@ def build_document(result: PipelineResult, model: str = "") -> list: + (f", Neustarts: {result.restarts}" if result.restarts else "")) if result.usage and result.usage.providers: meta.append("Anbieter: " + ", ".join(f"{name} ({n}×)" for name, n in result.usage.providers.items())) + if result.usage and result.usage.responded_models: + meta.append("Modell laut Antwort: " + ", ".join(f"{name} ({n}×)" for name, n in result.usage.responded_models.items())) doc.append(("list", meta)) doc += _result_section(result) diff --git a/tests/test_llm.py b/tests/test_llm.py index f117e61..5d2ee60 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -163,3 +163,29 @@ def test_understanding_and_figure_reading_share_their_own_model(monkeypatch): monkeypatch.setenv("LLM_MODEL_UNDERSTANDING", "bild/modell") assert model_for_step("Verstehen", "a/b") == "bild/modell" assert model_for_step("Abbildungen", "a/b") == "bild/modell" + + +# --- Tatsächlich antwortendes Modell (wichtig bei Aliasen wie ~deepseek/deepseek-flash-latest) --- + +def test_responded_model_is_recorded_per_call_and_summed(): + from types import SimpleNamespace as NS + session, _ = fake_session(model="~deepseek/deepseek-flash-latest") + + def antwort(modell): + return NS(provider="Together", model=modell, + usage=NS(prompt_tokens=10, completion_tokens=5, cost=0.0001, + completion_tokens_details=None, prompt_tokens_details=None)) + session._record(antwort("deepseek/deepseek-v4.1-flash"), step="Verstehen", model="~deepseek/deepseek-flash-latest") + session._record(antwort("deepseek/deepseek-v4.2-flash"), step="Extraktion", model="~deepseek/deepseek-flash-latest") + assert [(r.model, r.responded_model) for r in session.usage.call_records] == [ + ("~deepseek/deepseek-flash-latest", "deepseek/deepseek-v4.1-flash"), + ("~deepseek/deepseek-flash-latest", "deepseek/deepseek-v4.2-flash")] + assert session.usage.responded_models == {"deepseek/deepseek-v4.1-flash": 1, "deepseek/deepseek-v4.2-flash": 1} + + +def test_missing_or_odd_model_field_is_ignored(): + from types import SimpleNamespace as NS + session, _ = fake_session() + session._record(NS(provider=None, model=object(), usage=NS(prompt_tokens=1, completion_tokens=1, cost=None, + completion_tokens_details=None, prompt_tokens_details=None))) + assert session.usage.responded_models == {} and session.usage.call_records[0].responded_model is None diff --git a/tests/test_model_checks.py b/tests/test_model_checks.py index 6036fcd..5d2ef78 100644 --- a/tests/test_model_checks.py +++ b/tests/test_model_checks.py @@ -476,3 +476,22 @@ def test_bold_marker_with_colon_inside_is_recognised(): from checks.model_checks import QUESTION_PATTERN for zeile in ("**Teil 1:** Daten laden", "* **Question 2:** Solve", "**a)** Graph zeichnen"): assert QUESTION_PATTERN.search(zeile), zeile + + +def test_leading_zero_is_never_a_thousands_separator(): + from checks.model_checks import normalize_number + assert normalize_number("0.040") == "0.04" # vorher 40 (portfolio_qp) + assert normalize_number("1.234") == "1234" # deutsche Tausender unverändert + + +def test_latex_decimal_comma_is_one_number(): + from checks.model_checks import numbers_in + assert numbers_in(r"\mu_A=0{,}08, \sigma=0{,}040") == {"0.08", "0.04"} + + +def test_heading_markers_win_over_numbered_lists(): + from checks.model_checks import question_ids_in + text = ("## Restriktionen\n\n1. **Kassenbestand** je Periode\n\n2. **Lagerbilanz** je Produkt\n\n" + "### Teil 1: Formulierung\nKlassifizieren Sie das Problem.\n\n### Teil 2: Lösung\nLösen Sie das Modell.") + assert question_ids_in(text) == {"1", "2"} + assert question_ids_in("1. Berechnen Sie x.\n2. Deuten Sie y.\n3. Vergleichen Sie.") == {"1", "2", "3"}