dschlueter/text_to_solver_program
Watch
1
0
Fork
You've already forked text_to_solver_program
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: Prompt-Cache nutzbar und messbar; Grundsatzfrage Einzelaufruf gegen Stufenkette

Browse source 
OpenRouter-Aufrufe eines Laufs tragen eine gemeinsame session_id (Sticky Routing,
damit der Anbieter-Cache greifen kann); Cache-Treffer werden je Aufruf und Stufe
erfasst und im Log als Anteil ausgewiesen. docs/Konzeptionelle_Fragen_und_Antworten.md
hält Pipeline-Stufen und die Abwägung ein Aufruf vs. Kette mit Messdaten und Quellen fest.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter 2026-09-17 15:44:37 +02:00
commit eadf0608ce
2 changed files with 221 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

15
core/llm.py
View file

@ -8,6 +8,7 @@ import os
import sys
import threading
import time
import uuid
import instructor
@ -98,6 +99,10 @@ class LLMSession:
self.client = self._clients[self.provider]
self.usage = LLMUsage(model=model, provider=self.provider)
self._lock = threading.Lock()
# Eine Kennung je Pipeline-Lauf: OpenRouter leitet alle Aufrufe dann an denselben Anbieter (Sticky
# Routing), dessen Prompt-Cache wiederholte Präfixe billiger liest. Ohne sie wechselten die Anbieter
# je Aufruf (beobachtet: Modal, Together, Venice in einem Lauf) und kein Cache konnte greifen.
self.session_id = uuid.uuid4().hex
def overrides(self) -> dict:
"""Stufen mit abweichendem Modell – für die Fortschrittsausgabe."""
@ -113,7 +118,7 @@ class LLMSession:
kwargs["extra_body"] = {"provider": {
"require_parameters": True,
"preferred_min_throughput": {"p50": OPENROUTER_MIN_THROUGHPUT},
}}
}, "session_id": self.session_id}
if REASONING_EFFORT:
kwargs["extra_body"]["reasoning"] = {"effort": REASONING_EFFORT}
@ -130,7 +135,8 @@ class LLMSession:
time.sleep(wait)
def _record(self, completion, step: str = "unbenannt", seconds: float = 0.0, model: str = None) -> None:
before = (self.usage.completion_tokens, self.usage.reasoning_tokens)
before = (self.usage.completion_tokens, self.usage.reasoning_tokens, self.usage.prompt_tokens,
self.usage.cached_tokens)
with self._lock:
self.usage.calls += 1
provider = getattr(completion, "provider", None) # OpenRouter: tatsächlich antwortender Anbieter
@ -142,6 +148,8 @@ class LLMSession:
self.usage.completion_tokens += usage.completion_tokens or 0
details = getattr(usage, "completion_tokens_details", None)
self.usage.reasoning_tokens += getattr(details, "reasoning_tokens", 0) or 0
prompt_details = getattr(usage, "prompt_tokens_details", None)
self.usage.cached_tokens += getattr(prompt_details, "cached_tokens", 0) or 0
cost = getattr(usage, "cost", None)
if cost is not None:
self.usage.cost_usd = (self.usage.cost_usd or 0.0) + cost
@ -152,6 +160,7 @@ class LLMSession:
self.usage.prompt_tokens += meta.prompt_token_count or 0
self.usage.completion_tokens += meta.candidates_token_count or 0
self.usage.reasoning_tokens += meta.thoughts_token_count or 0
self.usage.cached_tokens += getattr(meta, "cached_content_token_count", 0) or 0
self._log_call(step, seconds, before, getattr(completion, "provider", None), model)
def _log_call(self, step, seconds, before, provider, model=None) -> None:
@ -159,4 +168,6 @@ class LLMSession:
step=step, seconds=round(seconds, 1), provider=provider, model=model,
completion_tokens=self.usage.completion_tokens - before[0],
reasoning_tokens=self.usage.reasoning_tokens - before[1],
prompt_tokens=self.usage.prompt_tokens - before[2],
cached_tokens=self.usage.cached_tokens - before[3],
))

8
core/model_schema.py
View file

@ -395,6 +395,8 @@ class CallRecord(BaseModel):
seconds: float
completion_tokens: int = 0
reasoning_tokens: int = 0
prompt_tokens: int = 0
cached_tokens: int = Field(default=0, description="Input-Tokens aus dem Anbieter-Cache (billiger)")
provider: Optional[str] = None
model: Optional[str] = Field(default=None, description="Tatsächlich benutztes Modell (kann je Stufe abweichen)")
@ -406,6 +408,7 @@ class LLMUsage(BaseModel):
prompt_tokens: int = 0
completion_tokens: int = Field(default=0, description="inkl. Reasoning-Tokens")
reasoning_tokens: int = 0
cached_tokens: int = Field(default=0, description="davon Input-Tokens aus dem Anbieter-Cache")
cost_usd: Optional[float] = Field(default=None, description="nur OpenRouter liefert Kosten")
providers: Dict[str, int] = Field(default_factory=dict, description="Antwortender Anbieter -> Anzahl Aufrufe (OpenRouter)")
call_records: List[CallRecord] = Field(default_factory=list)
@ -414,11 +417,14 @@ class LLMUsage(BaseModel):
"""Sekunden und Tokens je Pipeline-Stufe."""
summary: Dict[str, dict] = {}
for record in self.call_records:
entry = summary.setdefault(record.step, {"calls": 0, "seconds": 0.0, "completion_tokens": 0, "reasoning_tokens": 0})
entry = summary.setdefault(record.step, {"calls": 0, "seconds": 0.0, "completion_tokens": 0,
"reasoning_tokens": 0, "prompt_tokens": 0, "cached_tokens": 0})
entry["calls"] += 1
entry["seconds"] += record.seconds
entry["completion_tokens"] += record.completion_tokens
entry["reasoning_tokens"] += record.reasoning_tokens
entry["prompt_tokens"] += record.prompt_tokens
entry["cached_tokens"] += record.cached_tokens
return summary