youtube_abschrift_korrigieren/tests/test_aenderungen.py

"""Tests for aenderungen.py (change report raw transcript -> corrected text)."""
import html
import re
import sys
from pathlib import Path

import pytest

import aenderungen as ae


def changes(raw, corrected):
    return [(tag, " ".join(old), " ".join(new))
            for tag, old, new in ae.compare(raw, corrected).changes()]


def report_text(page):
    """Corrected text as shown in the report, without original/deleted words."""
    body = page.split("<h2>Korrigierter Text</h2>")[1]
    body = re.sub(r"<del[^>]*>.*?</del>|<span class=orig>.*?</span>", "", body)
    return html.unescape(re.sub(r"<[^>]+>", " ", body)).split()


@pytest.mark.parametrize("raw, corrected", [
    ("das ist gut", "Das ist gut."),                      # case, punctuation
    ("all das zerstritten", "All das – zerstritten."),    # dash only
    ("wie z.B. hier", "wie z. B. hier"),                  # split abbreviation
    ("eins zwei", "eins\n\nzwei"),                        # paragraph break
])
def test_expected_edits_are_no_changes(raw, corrected):
    assert changes(raw, corrected) == []


@pytest.mark.parametrize("raw, corrected", [
    ("gibt's", "gibt’s"),                                 # apostrophe variants
    ("er sagte Frieden", "Er sagte ‚Frieden‘."),          # quotes at the edges
])
def test_apostrophes_and_quotes_unchanged(raw, corrected):
    assert changes(raw, corrected) == []


def test_apostrophe_inside_word_is_a_change():
    assert changes("Egon Bahrs Politik", "Egon Bahr’s Politik") == [
        ("replace", "Bahrs", "Bahr’s"),
    ]


def test_english_review_marks():
    comparison = ae.compare("his name is Kocher", "His name is Kocher [check: Kocha].")
    assert list(comparison.changes()) == []
    assert sum(len(m) for m in comparison.marks.values()) == 1


def test_replace_insert_delete():
    raw = "die gedeiliche Zusammenarbeit war das gut heute"
    corrected = "die gedeihliche Zusammenarbeit war doch gut"
    assert changes(raw, corrected) == [
        ("replace", "gedeiliche", "gedeihliche"),
        ("replace", "das", "doch"),
        ("delete", "heute", ""),
    ]
    assert changes("ein Wort", "ein neues Wort") == [("insert", "", "neues")]


def test_change_summary_counts_and_sorts():
    raw = "Kremmel und Kremmel und Bar"
    corrected = "Kreml und Kreml und Bahr"
    assert ae.change_summary(ae.compare(raw, corrected)) == [
        ("Kremmel", "Kreml", 2), ("Bar", "Bahr", 1),
    ]


def test_review_marks_are_not_changes_but_shown():
    corrected = "Er heißt Gerhard [prüfen: Ghard] Kocher."
    comparison = ae.compare("Er heißt Gerhard Kocher", corrected)
    assert list(comparison.changes()) == []
    page = ae.render_html(comparison, "t")
    assert "<span class=check>[prüfen: Ghard]</span>" in page
    assert "1 Prüfmarken" in page


def test_render_html_shows_text_original_and_paragraphs():
    raw = "die gedeiliche Zusammenarbeit <b> ist gut heute neuer Absatz"
    corrected = "Die gedeihliche Zusammenarbeit <b> ist gut.\n\nNeuer Absatz."
    page = ae.render_html(ae.compare(raw, corrected), "Rede & Co")

    assert report_text(page) == corrected.split()
    assert "title='Original: gedeiliche'" in page
    assert "<del title='gestrichen'>heute</del>" in page
    assert "&lt;b&gt;" in page and "<b>" not in page.split("<body>")[1]
    assert "Rede &amp; Co" in page
    assert page.split("<h2>Korrigierter Text</h2>")[1].count("<p>") == 2


def test_report_path():
    assert ae.report_path(Path("a/rede.korrigiert.txt")) == Path("a/rede.aenderungen.html")
    assert ae.report_path(Path("out.txt")) == Path("out.aenderungen.html")


def test_cli(tmp_path, monkeypatch, capsys):
    raw = tmp_path / "rede.vtt"
    raw.write_text(
        "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\ndas Kremmel heute\n", encoding="utf-8"
    )
    corrected = tmp_path / "rede.korrigiert.txt"
    corrected.write_text("Das Kreml heute.\n", encoding="utf-8")
    monkeypatch.setattr(sys, "argv", ["aenderungen.py", "-a", str(raw), "-b", str(corrected)])
    ae.main()

    assert (tmp_path / "rede.aenderungen.html").is_file()
    out = capsys.readouterr().out
    assert "1× „Kremmel“ → „Kreml“" in out


def test_cli_missing_file(tmp_path, monkeypatch, capsys):
    monkeypatch.setattr(sys, "argv", ["aenderungen.py", "-a", "fehlt.txt", "-b", "fehlt.txt"])
    with pytest.raises(SystemExit):
        ae.main()
    assert "Datei nicht gefunden" in capsys.readouterr().err


# === Suspect changes ===

WORDS = {"professor", "das", "dass", "ist", "haus", "unterschreiben", "bar"}


def suspects(model_input, corrected, wordlist=WORDS, glossary=(), lang="de"):
    found = ae.review_candidates(ae.compare(model_input, corrected), wordlist, glossary, lang)
    return [(s.reason, s.old, s.new) for s in found]


@pytest.mark.parametrize("model_input, corrected, expected", [
    ("Der Professor kam", "Der Professorin kam", [(ae.VALID_WORD, "Professor", "Professorin")]),
    ("wir unterschreiben das", "wir unterscheiden das",
     [(ae.VALID_WORD, "unterschreiben", "unterscheiden")]),
    ("im CT1 heute", "im ZDF heute", [(ae.CHANGED_A_LOT, "CT1", "ZDF")]),
    ("er kam heute", "er ist kam heute", [(ae.INSERTED, "", "ist")]),
    ("er kam heute", "er kam", [(ae.DELETED, "heute", "")]),
])
def test_review_candidates_reports(model_input, corrected, expected):
    assert suspects(model_input, corrected) == expected


@pytest.mark.parametrize("model_input, corrected", [
    ("das Kremmel heute", "das Kreml heute"),                 # recognition error
    ("die Gradwanderung", "die Gratwanderung"),
    ("die Clan Kriminalität", "die Clankriminalität"),        # spacing only
    ("KI generierte Videos", "KI-generierte Videos"),         # hyphen only
    ("er äh kam ähm heute", "er kam heute"),                  # fillers
    ("das ist gut", "Das ist gut."),                          # case, punctuation
])
def test_review_candidates_ignores(model_input, corrected):
    assert suspects(model_input, corrected) == []


def test_review_candidates_glossary_spellings_are_not_reported():
    glossary = ae.glossary_entries(["Egon Bahr", ""], [("Bars", "Bahrs")])
    assert glossary == {("egon", "bahr"), ("bahrs",)}
    assert suspects("Bar sagte", "Bahr sagte", glossary=glossary) == []
    assert suspects("Bar sagte", "Bahr sagte") == [(ae.VALID_WORD, "Bar", "Bahr")]


@pytest.mark.parametrize("model_input, corrected, expected", [
    # A common word of a longer entry explains nothing on its own
    ("das sagt ihn nicht", "Das sagt Sie nicht.", [(ae.CHANGED_A_LOT, "ihn", "Sie")]),
    # Inserted words stay suspect even if they are in the glossary
    ("er kam heute", "er ist kam heute", [(ae.INSERTED, "", "ist")]),
    ("Dobrind sagte", "Alexander Dobrindt sagte",
     [(ae.CHANGED_A_LOT, "Dobrind", "Alexander Dobrindt")]),
])
def test_review_candidates_glossary_explains_only_spellings(model_input, corrected, expected):
    glossary = ae.glossary_entries(
        ["Alexander Dobrindt"], [("befinden Sie", "finden Sie"), ("is", "ist")]
    )
    assert suspects(model_input, corrected, glossary=glossary) == expected


def test_review_candidates_fillers_per_language():
    # "er" and "um" are real German words; in English they are fillers
    assert suspects("dann kam er um zu helfen", "Dann kam, zu helfen.") == [
        (ae.DELETED, "er um", "")
    ]
    english = suspects("so er we um go", "So we go.", lang="en")
    assert english == []


def test_review_candidates_without_wordlist():
    # Similar replacements of real words go unnoticed, the rest still counts
    assert suspects("Der Professor kam", "Der Professorin kam", wordlist=None) == []
    assert suspects("im CT1 heute", "im ZDF heute", wordlist=None) == [
        (ae.CHANGED_A_LOT, "CT1", "ZDF")
    ]


def test_review_candidates_indices_point_into_corrected_words():
    comparison = ae.compare("a b Professor c d", "A b Professorin c d.")
    [suspect] = ae.review_candidates(comparison, WORDS, set())
    assert comparison.words[suspect.j1:suspect.j2] == ["Professorin"]


def test_load_wordlist(capsys):
    assert "professor" in ae.load_wordlist("de")
    assert ae.load_wordlist("en") is None
    assert "keine Wortliste für „en“ (Paket wamerican)" in capsys.readouterr().out


def suspect_section(page):
    return page.split("<h2>Zu prüfen")[1].split("<h2>Alle Änderungen")[0]


def test_report_suspects_only_against_model_input(tmp_path, capsys):
    target = tmp_path / "bericht.html"
    # Glossary turned "Kremmel" into "Kreml" before the model; the model
    # replaced "Professor" and deleted "heute" on its own
    comparison = ae.write_report(
        "der Professor im Kremmel heute dort", "Der Professorin im Kreml dort.", target,
        "rede", model_input="der Professor im Kreml heute dort",
    )
    page = target.read_text(encoding="utf-8")
    section = suspect_section(page)

    assert [s.reason for s in comparison.suspects] == [ae.VALID_WORD, ae.DELETED]
    assert "Zu prüfen (2)" in page
    assert "Professorin" in section and "Kremmel" not in section
    assert "<span class='warn' title='gültiges Wort ersetzt: „Professor“ → „Professorin“'>" in page
    assert "<span class=warn title='gestrichen: „heute“ → „∅“'>⚠</span>" in page
    # Framing changes nothing about the text itself
    assert report_text(page) == "Der Professorin im Kreml ⚠ dort.".split()

    ae.print_summary(comparison)
    out = capsys.readouterr().out
    assert "Zu prüfen: 2 Stellen" in out
    assert "gültiges Wort ersetzt: „Professor“ → „Professorin“" in out


def test_report_without_suspects(tmp_path):
    target = tmp_path / "bericht.html"
    comparison = ae.write_report("das Kremmel", "Das Kreml.", target, "rede")
    assert comparison.suspects == []
    assert "Keine auffälligen Änderungen" in suspect_section(target.read_text(encoding="utf-8"))


def test_cli_ignores_time_headers_missing_in_raw(tmp_path, monkeypatch, capsys):
    raw = tmp_path / "rede.vtt"
    raw.write_text(
        "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\ndas Kremmel heute\n", encoding="utf-8"
    )
    corrected = tmp_path / "rede.korrigiert.txt"
    corrected.write_text("[00:00:01 - 00:00:02]\n\nDas Kreml heute.\n", encoding="utf-8")
    monkeypatch.setattr(sys, "argv", ["aenderungen.py", "-a", str(raw), "-b", str(corrected)])
    ae.main()
    out = capsys.readouterr().out
    assert "Zu prüfen: 0 Stellen" in out
    assert "00:00:01" not in out


def test_cli_glossary(tmp_path, monkeypatch, capsys):
    raw = tmp_path / "rede.txt"
    raw.write_text("das Kremmel und\nBar heute\n", encoding="utf-8")
    corrected = tmp_path / "rede.korrigiert.txt"
    corrected.write_text("Das Kreml und Bahr heute.\n", encoding="utf-8")
    glossary = tmp_path / "g.txt"
    glossary.write_text("Kremmel => Kreml\nEgon Bahr\n", encoding="utf-8")

    argv = ["aenderungen.py", "-a", str(raw), "-b", str(corrected)]
    monkeypatch.setattr(sys, "argv", argv)
    ae.main()
    assert "gültiges Wort ersetzt: „Bar“ → „Bahr“" in capsys.readouterr().out

    monkeypatch.setattr(sys, "argv", [*argv, "-g", str(glossary)])
    ae.main()
    out = capsys.readouterr().out
    assert "Zu prüfen: 0 Stellen" in out
    # The report still compares against the raw transcript
    assert "1× „Kremmel“ → „Kreml“" in out


@pytest.mark.parametrize("old, new, expected", [
    ("Punkten", "Punkte", True),
    ("das", "dass", True),
    ("sehe", "sähe", True),          # umlaut
    ("sein", "seien", True),         # both endings change
    ("kommt,", "kommen,", True),     # punctuation does not count
    ("Professor", "Professorin", False),   # "-in" is no inflection
    ("geworben", "geworden", False),
    ("unterschreiben", "unterscheiden", False),
    ("ich", "sich", False),
    ("den", "der", False),           # stem too short
    ("sich dazu", "zu", False),      # more than one word
])
def test_word_form_change(old, new, expected):
    assert ae.word_form_change(old.split(), new.split()) is expected


def test_word_form_change_english():
    assert ae.word_form_change(["walk"], ["walked"], "en")
    assert not ae.word_form_change(["walk"], ["walker"], "en")


def test_review_candidates_word_forms():
    words = WORDS | {"punkten", "punkte"}
    assert suspects("die Punkten und das", "die Punkte und dass", wordlist=words) == [
        (ae.WORD_FORM, "Punkten", "Punkte"), (ae.WORD_FORM, "das", "dass"),
    ]


def test_report_and_summary_put_word_forms_last(tmp_path, capsys):
    target = tmp_path / "bericht.html"
    comparison = ae.write_report(
        "das sagte der Professor", "dass sagte der Professorin", target, "rede"
    )
    assert [s.reason for s in comparison.suspects] == [ae.WORD_FORM, ae.VALID_WORD]
    page = target.read_text(encoding="utf-8")
    section = suspect_section(page)
    # Meaning change in the main table, the word form in its own group after it
    assert section.index("Professorin") < section.index("Wortform geändert (1)")
    assert section.index("Wortform geändert (1)") < section.index("<ins>dass</ins>")
    assert "<span class='warn form' title='Wortform geändert: „das“ → „dass“'>" in page

    ae.print_summary(comparison)
    out = capsys.readouterr().out.split("Zu prüfen: ")[1]
    assert out.startswith("2 Stellen (davon 1 Wortformen)")
    assert out.index("Professorin") < out.index("„das“ → „dass“")


def test_cli_uses_video_glossary(tmp_path, monkeypatch, capsys):
    folder = tmp_path / "rede"
    folder.mkdir()
    raw = folder / "rede.txt"
    raw.write_text("das Kremmel und\nBar heute\n", encoding="utf-8")
    corrected = folder / "rede.korrigiert.txt"
    corrected.write_text("Das Kreml und Bahr heute.\n", encoding="utf-8")
    (folder / "rede.glossar.txt").write_text("Kremmel => Kreml\nEgon Bahr\n", encoding="utf-8")

    monkeypatch.setattr(sys, "argv", ["aenderungen.py", "-a", str(raw), "-b", str(corrected)])
    ae.main()
    out = capsys.readouterr().out
    assert "Video-Glossar (automatisch)" in out
    assert "Zu prüfen: 0 Stellen" in out