dschlueter
101d55b713
- check_figure_readings vergleicht Zahlentupel je Datenzeile statt aller Zahlen des Freitexts (Fehlalarm im ersten echten Lauf: zusätzliche Knotenaufzählung); fängt falsch zugeordnete Bewertungen und vertauschte Pfeilrichtungen - feste Form 'Start → Ziel: Bewertung' in Verstehen- und Abbildungs-Prompt; abweichende Lesungen stehen im Log - Standardmodell ist der Alias ~deepseek/deepseek-flash-latest (derzeit V4.1 Flash); Benchmarks weiter mit festen IDs Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
152 lines
7.5 KiB
Python
152 lines
7.5 KiB
Python
"""Aufgaben mit Abbildungen (docling-Markdown): abtrennen, als Bild an die Verstehensstufe, doppelt ablesen."""
|
||
import base64
|
||
|
||
import pytest
|
||
|
||
from checks.model_checks import check_figure_readings, numbers_in
|
||
from core.model_schema import (FigureReading, FigureTranscription, SolverCode, SubQuestion, TaskUnderstanding)
|
||
from reporting.report import build_document, to_html, to_markdown
|
||
from TextToModel import run_pipeline
|
||
from understanding.figures import MAX_FIGURES, FigureError, split_figures, with_transcriptions
|
||
from understanding.understand import understand_task
|
||
|
||
# 1×1-PNG – Inhalt egal, es geht um den Weg durch die Pipeline
|
||
PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="
|
||
AUFGABE = f"Gesucht sind die minimalen Fahrtkosten von Knoten 1 nach Knoten 4.\n\n\n"
|
||
|
||
# Graph aus test_Image.md
|
||
GRAPH = FigureTranscription(id="Abbildung 1", shows="gerichteter Graph mit 4 Knoten",
|
||
transcription="Pfeil 1 → 2: 3\nPfeil 1 → 3: 1\nPfeil 3 → 2: 1\nPfeil 2 → 4: 3\nPfeil 3 → 4: 5")
|
||
VERLESEN = GRAPH.model_copy(update={"transcription": GRAPH.transcription.replace("2 → 4: 3", "2 → 4: 5")})
|
||
|
||
|
||
# --- Abtrennen ohne LLM ---
|
||
|
||
def test_embedded_image_is_replaced_by_a_placeholder():
|
||
text, figuren = split_figures(AUFGABE)
|
||
assert "[Abbildung 1]" in text and "base64" not in text
|
||
assert len(figuren) == 1 and figuren[0].mime == "image/png" and figuren[0].data == PNG
|
||
assert numbers_in(text) == {"1", "4"} # keine Zahlen aus dem Base64-Zeichensalat
|
||
|
||
|
||
def test_referenced_image_file_is_loaded_relative_to_the_task(tmp_path):
|
||
(tmp_path / "bilder").mkdir()
|
||
(tmp_path / "bilder" / "graph.png").write_bytes(base64.b64decode(PNG))
|
||
text, figuren = split_figures("Siehe  und  und ", tmp_path)
|
||
assert "[Abbildung 1: Netz]" in text
|
||
assert "" in text and "https://x.de/a.png" in text # nicht vorhanden bzw. keine Datei: bleibt
|
||
assert len(figuren) == 1 and figuren[0].data == PNG and figuren[0].source.endswith("graph.png")
|
||
|
||
|
||
def test_limits_and_broken_base64_are_errors_not_silent_cuts():
|
||
zu_viele = "\n".join(f"" for _ in range(MAX_FIGURES + 1))
|
||
with pytest.raises(FigureError, match="höchstens"):
|
||
split_figures(zu_viele)
|
||
with pytest.raises(FigureError, match="Base64"):
|
||
split_figures("")
|
||
|
||
|
||
def test_task_without_images_is_unchanged():
|
||
assert split_figures("Minimiere 3x + 2y.") == ("Minimiere 3x + 2y.", [])
|
||
|
||
|
||
def test_transcription_is_appended_with_uncertain_parts():
|
||
unsicher = GRAPH.model_copy(update={"uncertain": ["Richtung des Pfeils zwischen 2 und 3"]})
|
||
text = with_transcriptions("Aufgabe [Abbildung 1]", [unsicher])
|
||
assert "Pfeil 3 → 4: 5" in text and "Unsicher gelesen: Richtung des Pfeils" in text
|
||
|
||
|
||
# --- Nachricht an das Bildmodell ---
|
||
|
||
class Aufzeichner:
|
||
def __init__(self, antwort): self.antwort, self.nachrichten = antwort, []
|
||
def create(self, response_model, messages, step="unbenannt"):
|
||
self.nachrichten.append((step, messages)); return self.antwort
|
||
|
||
|
||
def test_understanding_sends_images_as_image_parts():
|
||
text, figuren = split_figures(AUFGABE)
|
||
session = Aufzeichner("ok")
|
||
understand_task(session, text, figures=figuren)
|
||
inhalt = session.nachrichten[0][1][-1]["content"]
|
||
assert isinstance(inhalt, list) and inhalt[0]["type"] == "text" and "1 ABBILDUNG" in inhalt[0]["text"]
|
||
assert inhalt[-1] == {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{PNG}"}}
|
||
|
||
|
||
def test_understanding_without_images_stays_plain_text():
|
||
session = Aufzeichner("ok")
|
||
understand_task(session, "Minimiere 3x.")
|
||
assert isinstance(session.nachrichten[0][1][-1]["content"], str)
|
||
|
||
|
||
# --- Doppelt ablesen ---
|
||
|
||
def test_identical_readings_have_no_finding():
|
||
assert check_figure_readings([GRAPH], [GRAPH.model_copy()]) == []
|
||
|
||
|
||
def test_misread_value_is_found_with_both_numbers():
|
||
befunde = check_figure_readings([GRAPH], [VERLESEN])
|
||
assert len(befunde) == 1 and befunde[0].code == "FIGURE_READINGS_DIFFER"
|
||
assert "nur in Lesung 1: (2 4 3)" in befunde[0].message and "nur in Lesung 2: (2 4 5)" in befunde[0].message
|
||
|
||
|
||
def test_figure_missing_in_one_reading_is_found():
|
||
assert "nur in einer" in check_figure_readings([GRAPH], [])[0].message
|
||
|
||
|
||
# --- Durch die ganze Pipeline (ohne LLM) ---
|
||
|
||
def _verstaendnis(figur):
|
||
return TaskUnderstanding(is_optimization_task=True, reason="-", facts=[], ignored_numbers=[],
|
||
sub_questions=[SubQuestion(id="1", quote="-", kind="rechnen", needs_solver=True)],
|
||
figures=[figur])
|
||
|
||
|
||
def _lauf(erste, lesungen):
|
||
from test_pipeline import GOOD, OK, FakeSession
|
||
from test_script_export import STRUCTURED_CODE
|
||
session = FakeSession([GOOD], [OK], [SolverCode(code=STRUCTURED_CODE)], understandings=[_verstaendnis(erste)],
|
||
readings=[FigureReading(figures=[f]) for f in lesungen])
|
||
return run_pipeline(AUFGABE, session=session, max_revisions=0, max_restarts=0), session
|
||
|
||
|
||
def test_only_the_understanding_sees_the_image_later_stages_get_the_transcription():
|
||
result, session = _lauf(GRAPH, [GRAPH])
|
||
bilder = [schritt for schritt, inhalt in session.contents if isinstance(inhalt, list)]
|
||
assert set(bilder) == {"Verstehen", "Abbildungen"}
|
||
spaeter = [inhalt for schritt, inhalt in session.contents if schritt not in ("Verstehen", "Abbildungen")]
|
||
assert spaeter and all("data:image" not in str(i) for i in spaeter)
|
||
assert any("Pfeil 3 → 4: 5" in str(i) for i in spaeter)
|
||
assert result.reliable and len(result.figures) == 1 and "Pfeil 2 → 4: 3" in result.task
|
||
|
||
|
||
def test_two_of_three_agreeing_readings_win():
|
||
result, _ = _lauf(VERLESEN, [GRAPH, GRAPH]) # erste Lesung verlesen, zweite und dritte stimmen überein
|
||
assert result.reliable and "Pfeil 2 → 4: 3" in result.task
|
||
|
||
|
||
def test_persistently_differing_readings_make_the_result_unreliable():
|
||
dritte = GRAPH.model_copy(update={"transcription": GRAPH.transcription.replace("1 → 3: 1", "1 → 3: 7")})
|
||
result, _ = _lauf(GRAPH, [VERLESEN, dritte])
|
||
assert not result.reliable
|
||
assert any(p.startswith("Abbildung:") and "uneinheitlich" in p for p in result.reliability_issues)
|
||
|
||
|
||
def test_report_shows_original_and_transcription():
|
||
result, _ = _lauf(GRAPH.model_copy(update={"uncertain": ["Pfeil 3 → 2 schwer lesbar"]}), [GRAPH])
|
||
html = to_html(build_document(result))
|
||
assert f'<img src="data:image/png;base64,{PNG}"' in html
|
||
assert "Pfeil 1 → 3: 1" in html and "unsicher gelesen" in html
|
||
assert f"" in to_markdown(build_document(result))
|
||
|
||
|
||
def test_reading_comparison_ignores_wording_but_not_assignment():
|
||
# erster echter Lauf: Knotenaufzählung und doppelte Pfeilbeschreibung lösten einen Fehlalarm aus
|
||
anders_formuliert = GRAPH.model_copy(update={"transcription": "(1,2): 3; (1,3): 1; (3,2): 1; (2,4): 3; (3,4): 5"})
|
||
assert check_figure_readings([GRAPH], [anders_formuliert]) == []
|
||
vertauscht = GRAPH.model_copy(update={"transcription": GRAPH.transcription.replace("1 → 2: 3", "1 → 2: 1")
|
||
.replace("1 → 3: 1", "1 → 3: 3")}) # gleiche Zahlen, falsche Pfeile
|
||
assert check_figure_readings([GRAPH], [vertauscht])
|
||
richtung = GRAPH.model_copy(update={"transcription": GRAPH.transcription.replace("3 → 2: 1", "2 → 3: 1")})
|
||
assert check_figure_readings([GRAPH], [richtung])
|