feat: kurze Absätze nach Inhalt mit dem passenden Nachbarn verbinden

Passt ein Absatz unter MIN_PARAGRAPH_SENTENCES an beide Nachbarn,
entscheidet das Modell, ob er den vorherigen Gedanken abschließt oder den
folgenden einleitet (JSON {"anschluss": "vorher"|"nachher"}); passt nur
einer, wird ohne Modell verbunden. Antworten werden in .parts/
zwischengespeichert. Der Cache und der JSON-Aufruf sind dafür als
cached_json() und chat_json() herausgezogen, vorhandene Caches gelten weiter.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter 2026-10-11 21:32:34 +02:00
commit 41bcd1eb7b

View file

@ -14,6 +14,14 @@ REPO = Path(__file__).resolve().parent.parent
SAMPLE = REPO / "testdaten" / "ramstein.de.txt"
GLOSSARY = REPO / "ramstein.glossar.txt"
SHARED_GLOSSARY = REPO / "krone-schmalz.glossar.txt"
# The autouse fixture below replaces it; tests of the real function use this
REAL_FIND_MERGE_SIDE = ak.find_merge_side
@pytest.fixture(autouse=True)
def merge_to_previous(monkeypatch):
"""No model call when joining short paragraphs: always the previous one."""
monkeypatch.setattr(ak, "find_merge_side", lambda *args: "vorher")
def flat(text):
@ -197,6 +205,40 @@ def test_find_paragraph_starts_bad_json(monkeypatch):
ak.find_paragraph_starts(["s"] * 3, 1, ak.PARAGRAPH_SYSTEM)
def merge_body(content):
return json.dumps({"message": {"content": content}}).encode("utf-8")
@pytest.mark.parametrize("side", ["vorher", "nachher"])
def test_find_merge_side(monkeypatch, side):
requests = []
def urlopen(request, timeout=None):
requests.append(json.loads(request.data))
return FakeResponse(merge_body(json.dumps({"anschluss": side})))
monkeypatch.setattr(ak.urllib.request, "urlopen", urlopen)
assert REAL_FIND_MERGE_SIDE("A.", "B.", "C.", ak.MERGE_SYSTEM) == side
payload = requests[0]
assert payload["format"] == ak.MERGE_FORMAT
assert payload["messages"][1]["content"] == "[vorher]\nA.\n\n[kurz]\nB.\n\n[nachher]\nC."
@pytest.mark.parametrize("content", ["kein JSON", '{"anschluss": "egal"}', "[]"])
def test_find_merge_side_bad_answer(monkeypatch, content):
monkeypatch.setattr(ak.urllib.request, "urlopen", fake_urlopen(merge_body(content)))
with pytest.raises(RuntimeError, match="unbrauchbare Antwort"):
REAL_FIND_MERGE_SIDE("A.", "B.", "C.", ak.MERGE_SYSTEM)
def test_cached_merge_side(tmp_path, monkeypatch):
monkeypatch.setattr(ak, "find_merge_side", lambda *args: "nachher")
prefix = tmp_path / "verbinden0005"
assert ak.cached_merge_side("A.", "B.", "C.", "sys", prefix) == ("nachher", False)
monkeypatch.setattr(ak, "find_merge_side", no_model_calls)
assert ak.cached_merge_side("A.", "B.", "C.", "sys", prefix) == ("nachher", True)
# === Sentences and paragraphs ===
def test_split_sentences_keeps_closing_quotes():
@ -242,7 +284,34 @@ def test_group_paragraphs():
([(1, 10), (11, 11)], [(1, 10), (11, 11)]), # no room anywhere
])
def test_merge_short_paragraphs(paragraphs, expected):
assert ak.merge_short_paragraphs(paragraphs) == expected
merged, sides = ak.merge_short_paragraphs(paragraphs)
assert merged == expected
assert sum(sides.values()) == len(paragraphs) - len(expected)
def test_merge_short_paragraphs_asks_only_with_two_options():
calls = []
def choose(previous, short, following):
calls.append((previous, short, following))
return "nachher"
paragraphs = [(1, 2), (3, 6), (7, 7), (8, 11), (12, 21), (22, 22), (23, 26)]
merged, sides = ak.merge_short_paragraphs(paragraphs, choose)
# (1, 2) can only join its follower; (22, 22) only its follower, the
# previous one is full; (7, 7) has room on both sides
assert calls == [((1, 6), (7, 7), (8, 11))]
assert merged == [(1, 6), (7, 11), (12, 21), (22, 26)]
assert sides == {"nachher": 3}
def test_merge_short_paragraphs_rechecks_joined_short_paragraph():
# Two single sentences in a row: the first joins the second, the pair is
# still too short and joins its neighbour
merged, _ = ak.merge_short_paragraphs(
[(1, 10), (11, 11), (12, 12), (13, 16)], lambda *args: "nachher"
)
assert merged == [(1, 10), (11, 16)]
def paragraph_lengths(text):
@ -287,9 +356,34 @@ def test_build_paragraphs_uses_cache(tmp_path, monkeypatch):
first = ak.build_paragraphs(text, tmp_path)
monkeypatch.setattr(ak, "find_paragraph_starts", no_model_calls)
monkeypatch.setattr(ak, "find_merge_side", no_model_calls)
assert ak.build_paragraphs(text, tmp_path) == first
def test_build_paragraphs_asks_where_short_paragraphs_belong(tmp_path, monkeypatch, capsys):
text = flat(SAMPLE.read_text(encoding="utf-8"))
# Paragraphs of 5 and 1 sentences alternate: every single one has room on both sides
monkeypatch.setattr(
ak, "find_paragraph_starts",
lambda sentences, first, system: {
n for n in range(max(first, 2), first + len(sentences)) if n % 6 in (1, 0)
},
)
asked = []
def find(previous, short, following, system):
asked.append(system)
return "nachher"
monkeypatch.setattr(ak, "find_merge_side", find)
result = ak.build_paragraphs(text, tmp_path)
assert asked and set(asked) == {ak.MERGE_SYSTEM}
assert flat(result) == text
assert "mit dem folgenden" in capsys.readouterr().out
assert min(paragraph_lengths(result)) >= ak.MIN_PARAGRAPH_SENTENCES
# === Command line and end-to-end ===
@pytest.fixture