feat: kurze Absätze nach Inhalt mit dem passenden Nachbarn verbinden
Passt ein Absatz unter MIN_PARAGRAPH_SENTENCES an beide Nachbarn,
entscheidet das Modell, ob er den vorherigen Gedanken abschließt oder den
folgenden einleitet (JSON {"anschluss": "vorher"|"nachher"}); passt nur
einer, wird ohne Modell verbunden. Antworten werden in .parts/
zwischengespeichert. Der Cache und der JSON-Aufruf sind dafür als
cached_json() und chat_json() herausgezogen, vorhandene Caches gelten weiter.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f1d2e201df
commit
41bcd1eb7b
1 changed files with 243 additions and 42 deletions
|
|
@ -14,6 +14,14 @@ REPO = Path(__file__).resolve().parent.parent
|
|||
SAMPLE = REPO / "testdaten" / "ramstein.de.txt"
|
||||
GLOSSARY = REPO / "ramstein.glossar.txt"
|
||||
SHARED_GLOSSARY = REPO / "krone-schmalz.glossar.txt"
|
||||
# The autouse fixture below replaces it; tests of the real function use this
|
||||
REAL_FIND_MERGE_SIDE = ak.find_merge_side
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def merge_to_previous(monkeypatch):
|
||||
"""No model call when joining short paragraphs: always the previous one."""
|
||||
monkeypatch.setattr(ak, "find_merge_side", lambda *args: "vorher")
|
||||
|
||||
|
||||
def flat(text):
|
||||
|
|
@ -197,6 +205,40 @@ def test_find_paragraph_starts_bad_json(monkeypatch):
|
|||
ak.find_paragraph_starts(["s"] * 3, 1, ak.PARAGRAPH_SYSTEM)
|
||||
|
||||
|
||||
def merge_body(content):
|
||||
return json.dumps({"message": {"content": content}}).encode("utf-8")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("side", ["vorher", "nachher"])
|
||||
def test_find_merge_side(monkeypatch, side):
|
||||
requests = []
|
||||
|
||||
def urlopen(request, timeout=None):
|
||||
requests.append(json.loads(request.data))
|
||||
return FakeResponse(merge_body(json.dumps({"anschluss": side})))
|
||||
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", urlopen)
|
||||
assert REAL_FIND_MERGE_SIDE("A.", "B.", "C.", ak.MERGE_SYSTEM) == side
|
||||
payload = requests[0]
|
||||
assert payload["format"] == ak.MERGE_FORMAT
|
||||
assert payload["messages"][1]["content"] == "[vorher]\nA.\n\n[kurz]\nB.\n\n[nachher]\nC."
|
||||
|
||||
|
||||
@pytest.mark.parametrize("content", ["kein JSON", '{"anschluss": "egal"}', "[]"])
|
||||
def test_find_merge_side_bad_answer(monkeypatch, content):
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", fake_urlopen(merge_body(content)))
|
||||
with pytest.raises(RuntimeError, match="unbrauchbare Antwort"):
|
||||
REAL_FIND_MERGE_SIDE("A.", "B.", "C.", ak.MERGE_SYSTEM)
|
||||
|
||||
|
||||
def test_cached_merge_side(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(ak, "find_merge_side", lambda *args: "nachher")
|
||||
prefix = tmp_path / "verbinden0005"
|
||||
assert ak.cached_merge_side("A.", "B.", "C.", "sys", prefix) == ("nachher", False)
|
||||
monkeypatch.setattr(ak, "find_merge_side", no_model_calls)
|
||||
assert ak.cached_merge_side("A.", "B.", "C.", "sys", prefix) == ("nachher", True)
|
||||
|
||||
|
||||
# === Sentences and paragraphs ===
|
||||
|
||||
def test_split_sentences_keeps_closing_quotes():
|
||||
|
|
@ -242,7 +284,34 @@ def test_group_paragraphs():
|
|||
([(1, 10), (11, 11)], [(1, 10), (11, 11)]), # no room anywhere
|
||||
])
|
||||
def test_merge_short_paragraphs(paragraphs, expected):
|
||||
assert ak.merge_short_paragraphs(paragraphs) == expected
|
||||
merged, sides = ak.merge_short_paragraphs(paragraphs)
|
||||
assert merged == expected
|
||||
assert sum(sides.values()) == len(paragraphs) - len(expected)
|
||||
|
||||
|
||||
def test_merge_short_paragraphs_asks_only_with_two_options():
|
||||
calls = []
|
||||
|
||||
def choose(previous, short, following):
|
||||
calls.append((previous, short, following))
|
||||
return "nachher"
|
||||
|
||||
paragraphs = [(1, 2), (3, 6), (7, 7), (8, 11), (12, 21), (22, 22), (23, 26)]
|
||||
merged, sides = ak.merge_short_paragraphs(paragraphs, choose)
|
||||
# (1, 2) can only join its follower; (22, 22) only its follower, the
|
||||
# previous one is full; (7, 7) has room on both sides
|
||||
assert calls == [((1, 6), (7, 7), (8, 11))]
|
||||
assert merged == [(1, 6), (7, 11), (12, 21), (22, 26)]
|
||||
assert sides == {"nachher": 3}
|
||||
|
||||
|
||||
def test_merge_short_paragraphs_rechecks_joined_short_paragraph():
|
||||
# Two single sentences in a row: the first joins the second, the pair is
|
||||
# still too short and joins its neighbour
|
||||
merged, _ = ak.merge_short_paragraphs(
|
||||
[(1, 10), (11, 11), (12, 12), (13, 16)], lambda *args: "nachher"
|
||||
)
|
||||
assert merged == [(1, 10), (11, 16)]
|
||||
|
||||
|
||||
def paragraph_lengths(text):
|
||||
|
|
@ -287,9 +356,34 @@ def test_build_paragraphs_uses_cache(tmp_path, monkeypatch):
|
|||
first = ak.build_paragraphs(text, tmp_path)
|
||||
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", no_model_calls)
|
||||
monkeypatch.setattr(ak, "find_merge_side", no_model_calls)
|
||||
assert ak.build_paragraphs(text, tmp_path) == first
|
||||
|
||||
|
||||
def test_build_paragraphs_asks_where_short_paragraphs_belong(tmp_path, monkeypatch, capsys):
|
||||
text = flat(SAMPLE.read_text(encoding="utf-8"))
|
||||
# Paragraphs of 5 and 1 sentences alternate: every single one has room on both sides
|
||||
monkeypatch.setattr(
|
||||
ak, "find_paragraph_starts",
|
||||
lambda sentences, first, system: {
|
||||
n for n in range(max(first, 2), first + len(sentences)) if n % 6 in (1, 0)
|
||||
},
|
||||
)
|
||||
asked = []
|
||||
|
||||
def find(previous, short, following, system):
|
||||
asked.append(system)
|
||||
return "nachher"
|
||||
|
||||
monkeypatch.setattr(ak, "find_merge_side", find)
|
||||
result = ak.build_paragraphs(text, tmp_path)
|
||||
|
||||
assert asked and set(asked) == {ak.MERGE_SYSTEM}
|
||||
assert flat(result) == text
|
||||
assert "mit dem folgenden" in capsys.readouterr().out
|
||||
assert min(paragraph_lengths(result)) >= ak.MIN_PARAGRAPH_SENTENCES
|
||||
|
||||
|
||||
# === Command line and end-to-end ===
|
||||
|
||||
@pytest.fixture
|
||||
|
|
|
|||
Loading…
Reference in a new issue