refactor: rede_korrigieren.py → abschrift_korrigieren.py, Texte neutral
Das Programm verarbeitet beliebige gesprochene Beiträge (Reden, Vorträge, Interviews, Videos). Prompts und Texte sprechen von „Beitrag“ statt „Rede“; README-Titel passend zum Projektverzeichnis youtube_abschrift_korrigieren. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
955bdc2659
commit
5a59baef88
3 changed files with 183 additions and 177 deletions
|
|
@ -13,8 +13,8 @@ def ollama_available(request, monkeypatch):
|
|||
if "real_ollama_check" in request.keywords:
|
||||
return
|
||||
import glossar_vorschlag
|
||||
import rede_korrigieren
|
||||
for module in (rede_korrigieren, glossar_vorschlag):
|
||||
import abschrift_korrigieren
|
||||
for module in (abschrift_korrigieren, glossar_vorschlag):
|
||||
monkeypatch.setattr(module, "check_ollama", lambda model: None)
|
||||
|
||||
|
||||
|
|
|
|||
728
tests/test_abschrift_korrigieren.py
Normal file
728
tests/test_abschrift_korrigieren.py
Normal file
|
|
@ -0,0 +1,728 @@
|
|||
"""Tests for abschrift_korrigieren.py; the Ollama model is always replaced by fakes."""
|
||||
import io
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import abschrift_korrigieren as ak
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
SAMPLE = REPO / "testdaten" / "ramstein.de.txt"
|
||||
GLOSSARY = REPO / "ramstein.glossar.txt"
|
||||
SHARED_GLOSSARY = REPO / "krone-schmalz.glossar.txt"
|
||||
|
||||
|
||||
def flat(text):
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
class FakeResponse(io.BytesIO):
|
||||
"""Stands in for urlopen(): iterable by lines, readable, context manager."""
|
||||
|
||||
|
||||
def fake_urlopen(body):
|
||||
def urlopen(request, timeout=None):
|
||||
return FakeResponse(body)
|
||||
return urlopen
|
||||
|
||||
|
||||
def identity_correct(chunk, index, system):
|
||||
return chunk.strip()
|
||||
|
||||
|
||||
def every_nth_start(n):
|
||||
"""Fake find_paragraph_starts: a paragraph starts every n sentences."""
|
||||
def find(sentences, first, system):
|
||||
last = first + len(sentences) - 1
|
||||
return {k for k in range(max(first, 2), last + 1) if (k - 1) % n == 0}
|
||||
return find
|
||||
|
||||
|
||||
def no_model_calls(*args, **kwargs):
|
||||
raise AssertionError("Modell wurde unerwartet aufgerufen")
|
||||
|
||||
|
||||
def run_main(monkeypatch, *args):
|
||||
monkeypatch.setattr(sys, "argv", ["abschrift_korrigieren.py", *map(str, args)])
|
||||
# main() sets the module-wide model; restore it after the test
|
||||
monkeypatch.setattr(ak, "MODEL", ak.MODEL)
|
||||
ak.main()
|
||||
|
||||
|
||||
# === Chunking and word counts ===
|
||||
|
||||
def test_split_chunks_keeps_text_and_ends_at_sentence():
|
||||
text = flat(SAMPLE.read_text(encoding="utf-8"))
|
||||
chunks = ak.split_chunks(text)
|
||||
|
||||
assert "".join(chunks) == text
|
||||
assert len(chunks) > 1
|
||||
for chunk in chunks[:-1]:
|
||||
assert ak.SENTENCE_END.search(chunk.rstrip())
|
||||
assert ak.CHUNK_SIZE <= len(chunk.split()) <= ak.CHUNK_SIZE + ak.MAX_EXTRA_WORDS
|
||||
|
||||
|
||||
def test_split_chunks_cuts_at_limit_without_sentence_end(monkeypatch):
|
||||
monkeypatch.setattr(ak, "CHUNK_SIZE", 3)
|
||||
monkeypatch.setattr(ak, "MAX_EXTRA_WORDS", 2)
|
||||
chunks = ak.split_chunks("a b c d e f g h")
|
||||
assert [len(c.split()) for c in chunks] == [5, 3]
|
||||
|
||||
|
||||
def test_split_chunks_empty():
|
||||
assert ak.split_chunks("") == []
|
||||
|
||||
|
||||
def test_count_words_ignores_review_marks():
|
||||
assert ak.count_words("Ramstein [prüfen: Rammstein] liegt") == 2
|
||||
|
||||
|
||||
# === Glossary ===
|
||||
|
||||
def test_read_glossary(tmp_path):
|
||||
path = tmp_path / "g.txt"
|
||||
path.write_text(
|
||||
"# Kommentar\n\nKremmel => Kreml\n Willly Brand => Willy Brandt \n"
|
||||
"Multi-Domain Task Force\nkaputt =>\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
terms, replacements = ak.read_glossary(path)
|
||||
assert terms == ["Multi-Domain Task Force"]
|
||||
assert replacements == [("Kremmel", "Kreml"), ("Willly Brand", "Willy Brandt")]
|
||||
|
||||
|
||||
def test_apply_replacements_whole_words_in_order(capsys):
|
||||
text = "Egon Bars Politik, Egon Bar. Barbara bleibt."
|
||||
result = ak.apply_replacements(
|
||||
text, [("Egon Bars", "Egon Bahrs"), ("Egon Bar", "Egon Bahr"), ("Fehlt", "x")]
|
||||
)
|
||||
assert result == "Egon Bahrs Politik, Egon Bahr. Barbara bleibt."
|
||||
assert "1 Ersetzung(en) ohne Treffer: „Fehlt“" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_build_system_appends_terms():
|
||||
assert ak.build_system([]) == ak.SYSTEM
|
||||
system = ak.build_system(["Kreml", "Egon Bahr"])
|
||||
assert system.startswith(ak.SYSTEM)
|
||||
assert "- Kreml\n- Egon Bahr" in system
|
||||
|
||||
|
||||
SAMPLE_GLOSSARIES = [
|
||||
("ramstein.glossar.txt", "ramstein.de.txt"),
|
||||
("magdeburg.glossar.txt", "magdeburg.de-orig.txt"),
|
||||
("lahnstein.glossar.txt", "Lahnstein.de-orig.txt"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("glossary, sample", SAMPLE_GLOSSARIES)
|
||||
def test_sample_glossary_replacements_all_match(capsys, glossary, sample):
|
||||
"""Every rule in a speech glossary must hit its speech, after the shared one."""
|
||||
text = flat((REPO / "testdaten" / sample).read_text(encoding="utf-8"))
|
||||
text = ak.apply_replacements(text, ak.read_glossary(SHARED_GLOSSARY)[1])
|
||||
capsys.readouterr()
|
||||
ak.apply_replacements(text, ak.read_glossary(REPO / glossary)[1])
|
||||
assert "ohne Treffer" not in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_shared_glossary_rules_match_some_speech():
|
||||
texts = [
|
||||
flat((REPO / "testdaten" / sample).read_text(encoding="utf-8"))
|
||||
for _, sample in SAMPLE_GLOSSARIES
|
||||
]
|
||||
for wrong, right in ak.read_glossary(SHARED_GLOSSARY)[1]:
|
||||
assert any(
|
||||
ak.apply_replacements(text, [(wrong, right)]) != text for text in texts
|
||||
), wrong
|
||||
|
||||
|
||||
# === Model calls (urlopen faked) ===
|
||||
|
||||
def stream_body(pieces, done_reason="stop"):
|
||||
lines = [{"message": {"content": p}, "done": False} for p in pieces]
|
||||
lines.append({"message": {"content": ""}, "done": True, "done_reason": done_reason,
|
||||
"eval_count": 10, "eval_duration": 1_000_000_000})
|
||||
return b"\n".join(json.dumps(line).encode("utf-8") for line in lines) + b"\n"
|
||||
|
||||
|
||||
def test_correct_chunk_joins_stream(monkeypatch, capsys):
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen",
|
||||
fake_urlopen(stream_body(["Ich grüße ", "Sie. ", "\n"])))
|
||||
assert ak.correct_chunk("ich grüße sie", 1, ak.SYSTEM) == "Ich grüße Sie."
|
||||
assert "10.0 Tokens/s" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_correct_chunk_truncated(monkeypatch):
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen",
|
||||
fake_urlopen(stream_body(["Text"], done_reason="length")))
|
||||
with pytest.raises(RuntimeError, match="abgeschnitten"):
|
||||
ak.correct_chunk("text", 1, ak.SYSTEM)
|
||||
|
||||
|
||||
def test_correct_chunk_empty(monkeypatch):
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", fake_urlopen(stream_body([" "])))
|
||||
with pytest.raises(RuntimeError, match="leere Modellantwort"):
|
||||
ak.correct_chunk("text", 1, ak.SYSTEM)
|
||||
|
||||
|
||||
def test_correct_chunk_ollama_error(monkeypatch):
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen",
|
||||
fake_urlopen(b'{"error": "model not found"}\n'))
|
||||
with pytest.raises(RuntimeError, match="model not found"):
|
||||
ak.correct_chunk("text", 1, ak.SYSTEM)
|
||||
|
||||
|
||||
def test_find_paragraph_starts_filters_numbers(monkeypatch):
|
||||
content = json.dumps({"absatzanfaenge": [1, 10, 12, 15, 20, "x"]})
|
||||
body = json.dumps({"message": {"content": content}}).encode("utf-8")
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", fake_urlopen(body))
|
||||
# Window holds sentences 10..15: 10 and 15 allowed, 1/20/"x" dropped
|
||||
assert ak.find_paragraph_starts(["s"] * 6, 10, ak.PARAGRAPH_SYSTEM) == {10, 12, 15}
|
||||
|
||||
|
||||
def test_find_paragraph_starts_never_first_sentence(monkeypatch):
|
||||
content = json.dumps({"absatzanfaenge": [1, 3]})
|
||||
body = json.dumps({"message": {"content": content}}).encode("utf-8")
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", fake_urlopen(body))
|
||||
assert ak.find_paragraph_starts(["s"] * 5, 1, ak.PARAGRAPH_SYSTEM) == {3}
|
||||
|
||||
|
||||
def test_find_paragraph_starts_bad_json(monkeypatch):
|
||||
body = json.dumps({"message": {"content": "kein JSON"}}).encode("utf-8")
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", fake_urlopen(body))
|
||||
with pytest.raises(RuntimeError, match="unbrauchbare Antwort"):
|
||||
ak.find_paragraph_starts(["s"] * 3, 1, ak.PARAGRAPH_SYSTEM)
|
||||
|
||||
|
||||
# === Sentences and paragraphs ===
|
||||
|
||||
def test_split_sentences_keeps_closing_quotes():
|
||||
text = "Er sagte: „Ja.“ Dann ging er. Wirklich?! [applaus] Gut… Ende."
|
||||
sentences = ak.split_sentences(text)
|
||||
assert sentences == ["Er sagte: „Ja.“", "Dann ging er.", "Wirklich?!",
|
||||
"[applaus] Gut…", "Ende."]
|
||||
assert " ".join(sentences) == text
|
||||
|
||||
|
||||
def test_group_paragraphs():
|
||||
assert ak.group_paragraphs(7, {3, 6}) == [(1, 2), (3, 5), (6, 7)]
|
||||
assert ak.group_paragraphs(3, set()) == [(1, 3)]
|
||||
assert ak.group_paragraphs(3, {1, 9}) == [(1, 3)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("paragraphs, expected", [
|
||||
([(1, 2), (3, 6)], [(1, 6)]), # short first joins next
|
||||
([(1, 4), (5, 5), (6, 9)], [(1, 5), (6, 9)]), # single joins previous
|
||||
([(1, 9), (10, 10), (11, 14)], [(1, 10), (11, 14)]), # fits exactly at maximum
|
||||
([(1, 10), (11, 11), (12, 15)], [(1, 10), (11, 15)]), # previous full: joins next
|
||||
([(1, 10), (11, 11)], [(1, 10), (11, 11)]), # no room anywhere
|
||||
])
|
||||
def test_merge_short_paragraphs(paragraphs, expected):
|
||||
assert ak.merge_short_paragraphs(paragraphs) == expected
|
||||
|
||||
|
||||
def paragraph_lengths(text):
|
||||
return [len(ak.split_sentences(p)) for p in text.split("\n\n")]
|
||||
|
||||
|
||||
def test_build_paragraphs_keeps_text_and_limits(tmp_path, monkeypatch):
|
||||
text = flat(SAMPLE.read_text(encoding="utf-8"))
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
|
||||
result = ak.build_paragraphs(text, tmp_path)
|
||||
|
||||
assert flat(result) == text
|
||||
lengths = paragraph_lengths(result)
|
||||
assert min(lengths) >= ak.MIN_PARAGRAPH_SENTENCES
|
||||
assert max(lengths) <= ak.MAX_PARAGRAPH_SENTENCES
|
||||
|
||||
|
||||
def test_build_paragraphs_splits_long_and_falls_back(tmp_path, monkeypatch, capsys):
|
||||
text = flat(SAMPLE.read_text(encoding="utf-8"))
|
||||
|
||||
def find(sentences, first, system):
|
||||
if system == ak.PARAGRAPH_SYSTEM:
|
||||
return every_nth_start(25)(sentences, first, system) # far too long
|
||||
if first < 100:
|
||||
return every_nth_start(6)(sentences, first, system) # split works
|
||||
return set() # split fails
|
||||
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", find)
|
||||
result = ak.build_paragraphs(text, tmp_path)
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert "Teile langen Absatz" in out
|
||||
assert "gleichmäßig" in out
|
||||
assert flat(result) == text
|
||||
assert max(paragraph_lengths(result)) <= ak.MAX_PARAGRAPH_SENTENCES
|
||||
|
||||
|
||||
def test_build_paragraphs_uses_cache(tmp_path, monkeypatch):
|
||||
text = flat(SAMPLE.read_text(encoding="utf-8"))
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
first = ak.build_paragraphs(text, tmp_path)
|
||||
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", no_model_calls)
|
||||
assert ak.build_paragraphs(text, tmp_path) == first
|
||||
|
||||
|
||||
# === Command line and end-to-end ===
|
||||
|
||||
@pytest.fixture
|
||||
def sample_copy(tmp_path):
|
||||
path = tmp_path / "ramstein.de.txt"
|
||||
shutil.copy(SAMPLE, path)
|
||||
return path
|
||||
|
||||
|
||||
def test_main_default_output_resume_and_glossary(sample_copy, monkeypatch, capsys):
|
||||
monkeypatch.setattr(ak, "correct_chunk", identity_correct)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
run_main(monkeypatch, "-i", sample_copy, "-g", SHARED_GLOSSARY, "-g", GLOSSARY)
|
||||
|
||||
target = sample_copy.with_name("ramstein.de.korrigiert.txt")
|
||||
result = target.read_text(encoding="utf-8")
|
||||
assert "Ramstein" in result and "Rammstein" not in result
|
||||
assert "Kreml" in result and "1983" in result
|
||||
assert (sample_copy.parent / "ramstein.de.korrigiert.parts").is_dir()
|
||||
assert "WARNUNG" not in capsys.readouterr().out
|
||||
|
||||
# Second run: everything cached, no model calls
|
||||
monkeypatch.setattr(ak, "correct_chunk", no_model_calls)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", no_model_calls)
|
||||
run_main(monkeypatch, "-i", sample_copy, "-g", SHARED_GLOSSARY, "-g", GLOSSARY)
|
||||
assert target.read_text(encoding="utf-8") == result
|
||||
|
||||
|
||||
def test_main_output_option_creates_directories(sample_copy, monkeypatch):
|
||||
monkeypatch.setattr(ak, "correct_chunk", identity_correct)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
target = sample_copy.parent / "neu" / "unter" / "rede.txt"
|
||||
run_main(monkeypatch, "-i", sample_copy, "-o", target)
|
||||
assert target.is_file()
|
||||
assert (target.parent / "rede.parts").is_dir()
|
||||
|
||||
|
||||
def test_main_vtt_input_matches_txt_input(sample_copy, monkeypatch):
|
||||
monkeypatch.setattr(ak, "correct_chunk", identity_correct)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
vtt = sample_copy.parent / "vtt" / "ramstein.de.vtt"
|
||||
vtt.parent.mkdir()
|
||||
shutil.copy(SAMPLE.with_suffix(".vtt"), vtt)
|
||||
|
||||
run_main(monkeypatch, "-i", sample_copy)
|
||||
run_main(monkeypatch, "-i", vtt)
|
||||
expected = sample_copy.with_name("ramstein.de.korrigiert.txt")
|
||||
assert vtt.with_name("ramstein.de.korrigiert.txt").read_text(
|
||||
encoding="utf-8"
|
||||
) == expected.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_main_writes_change_report(sample_copy, monkeypatch, capsys):
|
||||
def correct(chunk, index, system):
|
||||
return chunk.strip().replace("Rammstein", "Ramstein")
|
||||
|
||||
monkeypatch.setattr(ak, "correct_chunk", correct)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
run_main(monkeypatch, "-i", sample_copy)
|
||||
|
||||
report = sample_copy.with_name("ramstein.de.aenderungen.html")
|
||||
assert "title='Original: Rammstein" in report.read_text(encoding="utf-8")
|
||||
out = capsys.readouterr().out
|
||||
assert "„Rammstein“ → „Ramstein“" in out
|
||||
assert f"Bericht: {report}" in out
|
||||
|
||||
|
||||
def test_main_without_change_report(sample_copy, monkeypatch):
|
||||
monkeypatch.setattr(ak, "correct_chunk", identity_correct)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
run_main(monkeypatch, "-i", sample_copy, "--kein-bericht")
|
||||
assert not sample_copy.with_name("ramstein.de.aenderungen.html").exists()
|
||||
|
||||
|
||||
def test_main_model_option(sample_copy, monkeypatch):
|
||||
default = ak.MODEL
|
||||
used = []
|
||||
|
||||
def correct(chunk, index, system):
|
||||
used.append(ak.MODEL)
|
||||
return chunk.strip()
|
||||
|
||||
monkeypatch.setattr(ak, "correct_chunk", correct)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
parts = sample_copy.parent / "ramstein.de.korrigiert.parts"
|
||||
|
||||
run_main(monkeypatch, "-i", sample_copy)
|
||||
default_parts = {p.name for p in parts.glob("part*")}
|
||||
run_main(monkeypatch, "-i", sample_copy, "-m", "anderes:7b")
|
||||
|
||||
assert set(used) == {default, "anderes:7b"}
|
||||
# Another model must not reuse the cached parts of the default model
|
||||
assert {p.name for p in parts.glob("part*")} - default_parts
|
||||
|
||||
|
||||
def test_correct_chunk_sends_model(monkeypatch):
|
||||
sent = {}
|
||||
|
||||
def urlopen(request, timeout=None):
|
||||
sent.update(json.loads(request.data))
|
||||
return FakeResponse(stream_body(["Text."]))
|
||||
|
||||
monkeypatch.setattr(ak, "MODEL", "anderes:7b")
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", urlopen)
|
||||
ak.correct_chunk("Text.", 1, "system")
|
||||
assert sent["model"] == "anderes:7b"
|
||||
|
||||
|
||||
URL = "https://www.youtube.com/watch?v=abc123"
|
||||
VIDEO = {
|
||||
"id": "abc123", "title": "Rede", "channel": "Kanal", "upload_date": "20260328",
|
||||
"subtitles": {}, "automatic_captions": {"de-orig": []},
|
||||
}
|
||||
VTT = (
|
||||
"WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nDer Kremmel ist in Moskau.\n\n"
|
||||
"00:00:02.000 --> 00:00:03.000\nDer Kremmel ist in Moskau.\nDas ist bekannt.\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def from_url(fake_yt_dlp, yt_dlp_installed, monkeypatch):
|
||||
"""Fake download plus fake model for runs with -u."""
|
||||
monkeypatch.setattr(subprocess, "run", fake_yt_dlp(VIDEO, vtt=VTT))
|
||||
monkeypatch.setattr(ak, "correct_chunk", identity_correct)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
|
||||
|
||||
def test_main_url_default_names(from_url, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
run_main(monkeypatch, "-u", URL, "-g", SHARED_GLOSSARY)
|
||||
|
||||
raw = (tmp_path / "Kanal_Rede_2026-03-28.txt").read_text(encoding="utf-8")
|
||||
assert raw == "Der Kremmel ist in Moskau.\nDas ist bekannt.\n"
|
||||
result = (tmp_path / "Kanal_Rede_2026-03-28.korrigiert.txt").read_text(encoding="utf-8")
|
||||
assert result == "Der Kreml ist in Moskau. Das ist bekannt.\n"
|
||||
assert (tmp_path / "Kanal_Rede_2026-03-28.aenderungen.html").is_file()
|
||||
|
||||
|
||||
def test_main_url_output_option_and_kept_raw_text(from_url, tmp_path, monkeypatch, capsys):
|
||||
target = tmp_path / "out" / "rede.txt"
|
||||
raw_path = tmp_path / "out" / "rede.roh.txt"
|
||||
run_main(monkeypatch, "-u", URL, "-o", target)
|
||||
assert raw_path.is_file() and target.is_file()
|
||||
|
||||
# A raw text edited by hand is never overwritten
|
||||
raw_path.write_text("von Hand bearbeitet\n", encoding="utf-8")
|
||||
run_main(monkeypatch, "-u", URL, "-o", target)
|
||||
assert raw_path.read_text(encoding="utf-8") == "von Hand bearbeitet\n"
|
||||
assert "bleibt unverändert" in capsys.readouterr().out
|
||||
|
||||
|
||||
def recording_model(monkeypatch):
|
||||
"""Fake correction and paragraph models that record their system prompts."""
|
||||
systems = []
|
||||
|
||||
def correct(chunk, index, system):
|
||||
systems.append(system)
|
||||
return chunk.strip()
|
||||
|
||||
def find(sentences, first, system):
|
||||
systems.append(system)
|
||||
return set()
|
||||
|
||||
monkeypatch.setattr(ak, "correct_chunk", correct)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", find)
|
||||
return systems
|
||||
|
||||
|
||||
def test_build_system_english():
|
||||
system = ak.build_system(["Mearsheimer"], "en")
|
||||
assert system.startswith(ak.SYSTEM_EN)
|
||||
assert "Glossary:" in system and "- Mearsheimer" in system
|
||||
|
||||
|
||||
def test_main_english_prompts(sample_copy, monkeypatch, capsys):
|
||||
systems = recording_model(monkeypatch)
|
||||
run_main(monkeypatch, "-i", sample_copy, "-l", "en-US")
|
||||
assert systems[0] == ak.SYSTEM_EN
|
||||
assert ak.PARAGRAPH_SYSTEM_EN in systems
|
||||
assert "Sprache des Beitrags: en" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_main_default_language_is_german(sample_copy, monkeypatch):
|
||||
systems = recording_model(monkeypatch)
|
||||
run_main(monkeypatch, "-i", sample_copy)
|
||||
assert systems[0] == ak.SYSTEM
|
||||
assert ak.PARAGRAPH_SYSTEM in systems
|
||||
|
||||
|
||||
@pytest.mark.parametrize("language, expected", [("en-US", "SYSTEM_EN"), ("de", "SYSTEM")])
|
||||
def test_main_url_language_from_video(
|
||||
fake_yt_dlp, yt_dlp_installed, tmp_path, monkeypatch, language, expected
|
||||
):
|
||||
info = dict(VIDEO, language=language, automatic_captions={f"{language}-orig": []})
|
||||
monkeypatch.setattr(subprocess, "run", fake_yt_dlp(info, vtt=VTT))
|
||||
systems = recording_model(monkeypatch)
|
||||
run_main(monkeypatch, "-u", URL, "-o", tmp_path / "rede.korrigiert.txt")
|
||||
assert systems[0] == getattr(ak, expected)
|
||||
|
||||
|
||||
def test_main_url_unsupported_language(fake_yt_dlp, yt_dlp_installed, tmp_path, monkeypatch):
|
||||
info = dict(VIDEO, language="fr", automatic_captions={"fr-orig": []})
|
||||
monkeypatch.setattr(subprocess, "run", fake_yt_dlp(info, vtt=VTT))
|
||||
recording_model(monkeypatch)
|
||||
with pytest.raises(SystemExit, match="„fr“; unterstützt: de, en"):
|
||||
run_main(monkeypatch, "-u", URL, "-o", tmp_path / "rede.korrigiert.txt")
|
||||
|
||||
|
||||
def test_raw_text_path():
|
||||
assert ak.raw_text_path(Path("a/x.korrigiert.txt")) == Path("a/x.txt")
|
||||
assert ak.raw_text_path(Path("a/x.txt")) == Path("a/x.roh.txt")
|
||||
|
||||
|
||||
def test_main_warns_on_word_count_deviation(sample_copy, monkeypatch, capsys):
|
||||
def shortening(chunk, index, system):
|
||||
words = chunk.split()
|
||||
return " ".join(words[: int(len(words) * 0.9)]) if index == 2 else chunk.strip()
|
||||
|
||||
monkeypatch.setattr(ak, "correct_chunk", shortening)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
run_main(monkeypatch, "-i", sample_copy)
|
||||
out = capsys.readouterr().out
|
||||
assert "WARNUNG: Abschnitt 2:" in out
|
||||
assert "1 Abschnitt(e) mit auffälliger Wortzahl" in out
|
||||
|
||||
|
||||
def test_main_empty_input(tmp_path, monkeypatch):
|
||||
empty = tmp_path / "leer.txt"
|
||||
empty.write_text(" \n", encoding="utf-8")
|
||||
with pytest.raises(SystemExit, match="leer"):
|
||||
run_main(monkeypatch, "-i", empty)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args, message", [
|
||||
([], "required"),
|
||||
(["-i", "gibtsnicht.txt"], "Eingabedatei nicht gefunden"),
|
||||
(["-i", "{sample}", "-o", "{sample}"], "nicht die Eingabedatei"),
|
||||
(["-i", "{sample}", "-g", "gibtsnicht.txt"], "Glossar nicht gefunden"),
|
||||
(["-i", "{sample}", "-g", str(GLOSSARY), "-g", "fehlt.txt"], "Glossar nicht gefunden: fehlt.txt"),
|
||||
(["-i", "{sample}", "-u", "https://x"], "not allowed with"),
|
||||
(["-i", "{sample}", "-l", "fr"], "wird nicht unterstützt"),
|
||||
])
|
||||
def test_cli_errors(sample_copy, monkeypatch, capsys, args, message):
|
||||
args = [a.format(sample=sample_copy) for a in args]
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
run_main(monkeypatch, *args)
|
||||
assert exc.value.code == 2
|
||||
assert message in capsys.readouterr().err
|
||||
|
||||
|
||||
# === Time range (-s/-e) ===
|
||||
|
||||
def test_main_url_time_range_default_names(from_url, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
run_main(monkeypatch, "-u", URL, "-z", "0:00:02")
|
||||
|
||||
name = "Kanal_Rede_2026-03-28_ab_00-00-02"
|
||||
raw = (tmp_path / f"{name}.txt").read_text(encoding="utf-8")
|
||||
assert raw == "[00:00:02 - 00:00:03]\nDas ist bekannt.\n"
|
||||
result = (tmp_path / f"{name}.korrigiert.txt").read_text(encoding="utf-8")
|
||||
assert result == "[00:00:02 - 00:00:03]\n\nDas ist bekannt.\n"
|
||||
|
||||
|
||||
def test_main_url_range_from_beginning(from_url, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
run_main(monkeypatch, "-u", URL, "-z", "-0:00:02")
|
||||
raw = (tmp_path / "Kanal_Rede_2026-03-28_bis_00-00-02.txt").read_text(encoding="utf-8")
|
||||
assert raw == "[00:00:01 - 00:00:02]\nDer Kremmel ist in Moskau.\n"
|
||||
|
||||
|
||||
def test_main_url_time_range_without_text(from_url, tmp_path, monkeypatch):
|
||||
with pytest.raises(SystemExit, match="im gewählten Zeitraum keinen Text"):
|
||||
run_main(monkeypatch, "-u", URL, "-z", "1:00", "-o", tmp_path / "x.korrigiert.txt")
|
||||
|
||||
|
||||
def test_main_vtt_time_range(tmp_path, monkeypatch):
|
||||
vtt = tmp_path / "rede.de.vtt"
|
||||
vtt.write_text(VTT, encoding="utf-8")
|
||||
monkeypatch.setattr(ak, "correct_chunk", identity_correct)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
run_main(monkeypatch, "-i", vtt, "-z", "0-0:00:02")
|
||||
|
||||
result = tmp_path / "rede.de_bis_00-00-02.korrigiert.txt"
|
||||
assert result.read_text(encoding="utf-8") == (
|
||||
"[00:00:01 - 00:00:02]\n\nDer Kremmel ist in Moskau.\n"
|
||||
)
|
||||
|
||||
|
||||
def test_main_vtt_time_range_without_text(tmp_path, monkeypatch):
|
||||
vtt = tmp_path / "rede.de.vtt"
|
||||
vtt.write_text(VTT, encoding="utf-8")
|
||||
monkeypatch.setattr(ak, "correct_chunk", no_model_calls)
|
||||
with pytest.raises(SystemExit, match="im gewählten Zeitraum keinen Text"):
|
||||
run_main(monkeypatch, "-i", vtt, "-z", "1:00")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("args, message", [
|
||||
(["-i", "{sample}", "-z", "1:00"], "nur mit .vtt-Eingabe oder -u"),
|
||||
(["-i", "{sample}", "-z", "x"], "ungültiger Bereich"),
|
||||
(["-i", "{sample}", "-z", "2:00-1:00"], "Start muss vor Ende"),
|
||||
(["-i", "{sample}", "-z", "0:05 0:10"], "überschneiden"),
|
||||
])
|
||||
def test_cli_time_errors(sample_copy, monkeypatch, capsys, args, message):
|
||||
args = [a.format(sample=sample_copy) for a in args]
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
run_main(monkeypatch, *args)
|
||||
assert exc.value.code == 2
|
||||
assert message in capsys.readouterr().err
|
||||
|
||||
|
||||
# === Ollama availability ===
|
||||
|
||||
def tags_urlopen(*names):
|
||||
body = json.dumps({"models": [{"name": n} for n in names]}).encode("utf-8")
|
||||
return fake_urlopen(body)
|
||||
|
||||
|
||||
@pytest.mark.real_ollama_check
|
||||
@pytest.mark.parametrize("model", ["qwen3:32b", "llama3"])
|
||||
def test_check_ollama_model_installed(monkeypatch, model):
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", tags_urlopen("qwen3:32b", "llama3:latest"))
|
||||
ak.check_ollama(model)
|
||||
|
||||
|
||||
@pytest.mark.real_ollama_check
|
||||
def test_check_ollama_model_missing(monkeypatch):
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", tags_urlopen("qwen3:8b"))
|
||||
with pytest.raises(SystemExit, match="ollama pull qwen3:32b"):
|
||||
ak.check_ollama("qwen3:32b")
|
||||
|
||||
|
||||
def refused(request, timeout=None):
|
||||
raise ak.urllib.error.URLError(ConnectionRefusedError(111, "Connection refused"))
|
||||
|
||||
|
||||
@pytest.mark.real_ollama_check
|
||||
def test_main_ollama_down_stops_before_download(fake_yt_dlp, yt_dlp_installed, monkeypatch):
|
||||
run = fake_yt_dlp(VIDEO, vtt=VTT)
|
||||
monkeypatch.setattr(subprocess, "run", run)
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", refused)
|
||||
with pytest.raises(SystemExit, match="Ollama nicht erreichbar.*systemctl start ollama"):
|
||||
run_main(monkeypatch, "-u", URL)
|
||||
assert run.calls == []
|
||||
|
||||
|
||||
def test_correct_chunk_ollama_gone_midway(monkeypatch):
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", refused)
|
||||
with pytest.raises(SystemExit, match="Ollama nicht erreichbar"):
|
||||
ak.correct_chunk("Text.", 1, "system")
|
||||
|
||||
|
||||
def test_ollama_open_passes_http_errors(monkeypatch):
|
||||
def not_found(request, timeout=None):
|
||||
raise ak.urllib.error.HTTPError(ak.OLLAMA_URL, 404, "Not Found", {}, None)
|
||||
monkeypatch.setattr(ak.urllib.request, "urlopen", not_found)
|
||||
with pytest.raises(ak.urllib.error.HTTPError):
|
||||
ak.ollama_open(ak.OLLAMA_URL)
|
||||
|
||||
|
||||
# === Excerpts: sentence bounds and prompt note ===
|
||||
|
||||
@pytest.mark.parametrize("lang", ["de", "en"])
|
||||
def test_build_system_excerpt_note(lang):
|
||||
prompts = ak.PROMPTS[lang]
|
||||
assert prompts["excerpt"] not in ak.build_system(["X"], lang)
|
||||
system = ak.build_system(["X"], lang, excerpt=True)
|
||||
assert system.startswith(prompts["system"])
|
||||
assert system.index(prompts["excerpt"]) < system.index(prompts["glossary"])
|
||||
assert ak.build_system([], lang, excerpt=True).endswith(prompts["excerpt"] + "\n")
|
||||
|
||||
|
||||
CUT_VTT = (
|
||||
"WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nDer Kremmel ist\n\n"
|
||||
"00:00:02.000 --> 00:00:03.000\nin Moskau. Das ist\n\n"
|
||||
"00:00:03.000 --> 00:00:04.000\nbekannt.\n"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exact, raw", [
|
||||
([], "[00:00:01 - 00:00:04]\nDer Kremmel ist\nin Moskau. Das ist\nbekannt.\n"),
|
||||
(["--exakt"], "[00:00:02 - 00:00:03]\nin Moskau. Das ist\n"),
|
||||
])
|
||||
def test_main_url_excerpt(fake_yt_dlp, yt_dlp_installed, tmp_path, monkeypatch, exact, raw):
|
||||
monkeypatch.setattr(subprocess, "run", fake_yt_dlp(VIDEO, vtt=CUT_VTT))
|
||||
systems = recording_model(monkeypatch)
|
||||
target = tmp_path / "rede.korrigiert.txt"
|
||||
run_main(monkeypatch, "-u", URL, "-z", "0:00:02-0:00:03", "-o", target, *exact)
|
||||
|
||||
assert (tmp_path / "rede.txt").read_text(encoding="utf-8") == raw
|
||||
assert ak.PROMPTS["de"]["excerpt"] in systems[0]
|
||||
|
||||
|
||||
# === Several time ranges (-z): blocks with headers ===
|
||||
|
||||
MULTI_VTT = (
|
||||
"WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nDer Kremmel ist in Moskau.\n\n"
|
||||
"00:01:00.000 --> 00:01:01.000\nZwischendurch etwas anderes.\n\n"
|
||||
"00:02:00.000 --> 00:02:01.000\nDas ist bekannt.\n"
|
||||
)
|
||||
|
||||
|
||||
def chunk_recorder(monkeypatch):
|
||||
"""Fake models; returns the chunks and system prompts sent for correction."""
|
||||
sent = []
|
||||
|
||||
def correct(chunk, index, system):
|
||||
sent.append((chunk, system))
|
||||
return chunk.strip()
|
||||
|
||||
monkeypatch.setattr(ak, "correct_chunk", correct)
|
||||
monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5))
|
||||
return sent
|
||||
|
||||
|
||||
def test_main_url_several_ranges(fake_yt_dlp, yt_dlp_installed, tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.setattr(subprocess, "run", fake_yt_dlp(VIDEO, vtt=MULTI_VTT))
|
||||
sent = chunk_recorder(monkeypatch)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
run_main(monkeypatch, "-u", URL, "-z", "0:00:00-0:00:30 0:02", "-g", SHARED_GLOSSARY)
|
||||
|
||||
name = "Kanal_Rede_2026-03-28_bis_00-00-30+ab_00-02-00"
|
||||
assert (tmp_path / f"{name}.txt").read_text(encoding="utf-8") == (
|
||||
"[00:00:01 - 00:00:02]\nDer Kremmel ist in Moskau.\n\n"
|
||||
"[00:02:00 - 00:02:01]\nDas ist bekannt.\n"
|
||||
)
|
||||
assert (tmp_path / f"{name}.korrigiert.txt").read_text(encoding="utf-8") == (
|
||||
"[00:00:01 - 00:00:02]\n\nDer Kreml ist in Moskau.\n\n"
|
||||
"[00:02:00 - 00:02:01]\n\nDas ist bekannt.\n"
|
||||
)
|
||||
# One chunk per block, never a header; excerpt note in the prompt
|
||||
assert [chunk for chunk, _ in sent] == ["Der Kreml ist in Moskau.", "Das ist bekannt."]
|
||||
assert all(ak.PROMPTS["de"]["excerpt"] in system for _, system in sent)
|
||||
out = capsys.readouterr().out
|
||||
assert "Abschnitt 2/2" in out
|
||||
# Headers are no change; only the glossary replacement is
|
||||
assert "Änderungen gegenüber dem Rohtext: 1" in out
|
||||
|
||||
|
||||
def test_main_txt_with_headers_corrected_in_blocks(tmp_path, monkeypatch):
|
||||
raw = tmp_path / "rede.txt"
|
||||
raw.write_text(
|
||||
"[00:00:01 - 00:00:02]\nErster\nBlock.\n\n[00:02:00 - 00:02:01]\nZweiter Block.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
sent = chunk_recorder(monkeypatch)
|
||||
run_main(monkeypatch, "-i", raw)
|
||||
|
||||
assert [chunk for chunk, _ in sent] == ["Erster Block.", "Zweiter Block."]
|
||||
assert (tmp_path / "rede.korrigiert.txt").read_text(encoding="utf-8") == (
|
||||
"[00:00:01 - 00:00:02]\n\nErster Block.\n\n[00:02:00 - 00:02:01]\n\nZweiter Block.\n"
|
||||
)
|
||||
|
||||
|
||||
def test_main_without_headers_has_no_excerpt_note(sample_copy, monkeypatch):
|
||||
sent = chunk_recorder(monkeypatch)
|
||||
run_main(monkeypatch, "-i", sample_copy)
|
||||
assert ak.PROMPTS["de"]["excerpt"] not in sent[0][1]
|
||||
|
|
@ -7,7 +7,7 @@ import urllib.request
|
|||
import pytest
|
||||
|
||||
import glossar_vorschlag as gv
|
||||
import rede_korrigieren as rk
|
||||
import abschrift_korrigieren as ak
|
||||
|
||||
RAW = (
|
||||
"Der Kremmel und Egon Bar trafen Host Telchik. Die Rechtsstaatbahn war gut. "
|
||||
|
|
@ -114,12 +114,12 @@ def test_drafted_entries_are_valid_glossary(raw_file, monkeypatch, tmp_path):
|
|||
monkeypatch.setattr(urllib.request, "urlopen", fake_model(ENTRIES[:2]))
|
||||
out = tmp_path / "entwurf.txt"
|
||||
run_main(monkeypatch, "-i", raw_file, "-o", out)
|
||||
assert rk.read_glossary(out) == ([], []) # nothing active
|
||||
assert ak.read_glossary(out) == ([], []) # nothing active
|
||||
|
||||
# Removing "# ? " turns the lines into glossary entries
|
||||
accepted = tmp_path / "glossar.txt"
|
||||
accepted.write_text(out.read_text(encoding="utf-8").replace("# ? ", ""), encoding="utf-8")
|
||||
_, replacements = rk.read_glossary(accepted)
|
||||
_, replacements = ak.read_glossary(accepted)
|
||||
assert replacements == [("Grotzki", "Grotzky"), ("Ghard Kocher", "Gerhard Kocher")]
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue