diff --git a/.gitignore b/.gitignore index 3581fd6..9095b07 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,11 @@ __pycache__/ # Eigene Ergebnisse und Testläufe output/ +# Arbeitsordner je Video, fertige Abschriften und Quellenangaben: nur lokal +abschriften/ +fertig/ +*.quelle.json + # Echte Reden (Urheberrecht): nur lokal; die Tests nutzen tests/daten/ testdaten/ diff --git a/abschrift_korrigieren.py b/abschrift_korrigieren.py index f8c3f69..e051cb9 100755 --- a/abschrift_korrigieren.py +++ b/abschrift_korrigieren.py @@ -21,7 +21,9 @@ from vtt2txt import ( add_time_options, join_segments, range_argv, resolve_time_ranges, split_segments, time_suffix, vtt_to_text, ) -from yt_untertitel import YT_DLP, base_language, default_name, original_language +from yt_untertitel import ( + YT_DLP, base_language, default_name, original_language, work_folder, write_source, +) MODEL = "qwen3:32b" OLLAMA_URL = "http://127.0.0.1:11434/api/chat" @@ -264,6 +266,22 @@ def read_glossary(path): return terms, replacements +def video_glossary(folder): + """Glossary of the video in `folder`, named like the folder: + abschriften//.glossar.txt; None if there is none.""" + path = folder / f"{folder.resolve().name}.glossar.txt" + return path if path.is_file() else None + + +def with_video_glossary(glossaries, folder): + """The given glossaries plus the video glossary of `folder` (last, after the + shared ones), unless it was given; returns (glossaries, added or None).""" + found = video_glossary(folder) + if found is None or any(g.resolve() == found.resolve() for g in glossaries): + return list(glossaries), None + return [*glossaries, found], found + + def apply_replacements(text, replacements): """Replace whole words in file order; report rules without any match.""" unmatched = [] @@ -633,8 +651,8 @@ def parse_args(): parser.add_argument( "-o", "--output", type=Path, help="Ausgabedatei (Standard: .korrigiert.txt, " - "mit -u __.korrigiert.txt; mit -z zusätzlich " - "die Bereiche)", + "mit -u abschriften//.korrigiert.txt mit " + " = __; mit -z zusätzlich die Bereiche)", ) parser.add_argument( "-g", "--glossar", type=Path, action="append", default=[], @@ -709,9 +727,12 @@ def read_source(args): f"Der Beitrag ist auf „{lang}“; unterstützt: {', '.join(PROMPTS)} (-l)." ) suffix = time_suffix(args.bereiche) - target = args.output or Path(f"{default_name(info)}{suffix}.korrigiert.txt") + name = default_name(info) + target = args.output or work_folder(name) / f"{name}{suffix}.korrigiert.txt" raw = text + "\n" - save_raw_text(raw_text_path(target), raw) + raw_path = raw_text_path(target) + save_raw_text(raw_path, raw) + write_source(info, raw_path.with_suffix(".quelle.json")) return raw, target, lang raw = args.input.read_text(encoding="utf-8-sig") @@ -762,10 +783,12 @@ def main(): parts_dir = target.with_suffix(".parts") terms, replacements = [], [] - for glossary in args.glossar: + glossaries, automatic = with_video_glossary(args.glossar, target.parent) + for glossary in glossaries: more_terms, more_replacements = read_glossary(glossary) + note = " (Video-Glossar, automatisch)" if glossary == automatic else "" print( - f"Glossar {glossary}: {len(more_terms)} Begriffe, " + f"Glossar {glossary}{note}: {len(more_terms)} Begriffe, " f"{len(more_replacements)} feste Ersetzungen" ) terms += more_terms diff --git a/aenderungen.py b/aenderungen.py index f2eca03..32af953 100755 --- a/aenderungen.py +++ b/aenderungen.py @@ -473,13 +473,16 @@ def parse_args(): def main(): # abschrift_korrigieren imports this module: import it only when needed - from abschrift_korrigieren import apply_replacements, read_glossary + from abschrift_korrigieren import apply_replacements, read_glossary, with_video_glossary from yt_untertitel import base_language args = parse_args() raw = read_raw(args.roh) terms, replacements = [], [] - for glossary in args.glossar: + glossaries, automatic = with_video_glossary(args.glossar, args.korrigiert.parent) + if automatic is not None: + print(f"Video-Glossar (automatisch): {automatic}") + for glossary in glossaries: more_terms, more_replacements = read_glossary(glossary) terms += more_terms replacements += more_replacements diff --git a/extract_text.py b/extract_text.py index a4c5a76..1ba11a9 100755 --- a/extract_text.py +++ b/extract_text.py @@ -14,7 +14,7 @@ from vtt2txt import ( ) from yt_untertitel import ( YT_DLP, choose_track, default_name, download, fetch_info, no_track_message, - original_language, + original_language, work_folder, write_source, ) FLOW_WIDTH = 80 # Zeilenbreite für --fliesstext @@ -31,8 +31,8 @@ def parse_args(): ) parser.add_argument( "-o", "--output", type=Path, - help="Ausgabedatei (Standard: __.txt, " - "mit -z zusätzlich die Bereiche)", + help="Ausgabedatei (Standard: abschriften//.txt mit " + " = __, mit -z zusätzlich die Bereiche)", ) parser.add_argument( "-f", "--fliesstext", action="store_true", @@ -87,7 +87,8 @@ def main(): args = parse_args() info, text = fetch_text(args.url, args.lang, args.bereiche, args.exakt) suffix = time_suffix(args.bereiche) - output = args.output or Path(f"{default_name(info)}{suffix}.txt") + name = default_name(info) + output = args.output or work_folder(name) / f"{name}{suffix}.txt" segments = split_segments(text) if args.fliesstext: @@ -104,6 +105,7 @@ def main(): output.parent.mkdir(parents=True, exist_ok=True) output.write_text(text + "\n", encoding="utf-8") + write_source(info, output.with_suffix(".quelle.json")) print( f"Gespeichert: {output} " f"({len(text.splitlines())} Zeilen, " diff --git a/glossar_vorschlag.py b/glossar_vorschlag.py index 2750478..28d532d 100755 --- a/glossar_vorschlag.py +++ b/glossar_vorschlag.py @@ -13,7 +13,7 @@ import re from aenderungen import compare, read_raw from abschrift_korrigieren import ( MODEL, apply_replacements, chat_request, check_ollama, ollama_open, read_glossary, - split_chunks, + split_chunks, with_video_glossary, ) from vtt2txt import split_segments @@ -226,7 +226,10 @@ def main(): raw = text known, replacements = set(), [] - for glossary in args.glossar: + glossaries, automatic = with_video_glossary(args.glossar, args.input.parent) + if automatic is not None: + print(f"Video-Glossar (automatisch): {automatic}") + for glossary in glossaries: _, more = read_glossary(glossary) replacements += more known.update(wrong for wrong, _ in more) diff --git a/tests/test_abschrift_korrigieren.py b/tests/test_abschrift_korrigieren.py index 297e8d2..1783ce7 100644 --- a/tests/test_abschrift_korrigieren.py +++ b/tests/test_abschrift_korrigieren.py @@ -636,13 +636,63 @@ def kreml_glossary(folder): def test_main_url_default_names(from_url, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) + work = tmp_path / "abschriften" / "Kanal_Rede_2026-03-28" run_main(monkeypatch, "-u", URL, "-g", kreml_glossary(tmp_path)) - raw = (tmp_path / "Kanal_Rede_2026-03-28.txt").read_text(encoding="utf-8") + raw = (work / "Kanal_Rede_2026-03-28.txt").read_text(encoding="utf-8") assert raw == "Der Kremmel ist in Moskau.\nDas ist bekannt.\n" - result = (tmp_path / "Kanal_Rede_2026-03-28.korrigiert.txt").read_text(encoding="utf-8") + result = (work / "Kanal_Rede_2026-03-28.korrigiert.txt").read_text(encoding="utf-8") assert result == "Der Kreml ist in Moskau. Das ist bekannt.\n" - assert (tmp_path / "Kanal_Rede_2026-03-28.aenderungen.html").is_file() + assert (work / "Kanal_Rede_2026-03-28.aenderungen.html").is_file() + source = json.loads((work / "Kanal_Rede_2026-03-28.quelle.json").read_text(encoding="utf-8")) + assert source["titel"] == "Rede" and source["kanal"] == "Kanal" + assert source["datum"] == "20260328" + assert source["url"] == "https://www.youtube.com/watch?v=abc123" + + +def video_folder(tmp_path): + """abschriften/rede/ with raw text and its video glossary.""" + folder = tmp_path / "abschriften" / "rede" + folder.mkdir(parents=True) + (folder / "rede.txt").write_text("Der Kremmel ist in Moskau.\n", encoding="utf-8") + (folder / "rede.glossar.txt").write_text("Kremmel => Kreml\n", encoding="utf-8") + return folder + + +def test_main_uses_video_glossary(tmp_path, monkeypatch, capsys): + folder = video_folder(tmp_path) + monkeypatch.setattr(ak, "correct_chunk", identity_correct) + monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5)) + run_main(monkeypatch, "-i", folder / "rede.txt") + + result = (folder / "rede.korrigiert.txt").read_text(encoding="utf-8") + assert result == "Der Kreml ist in Moskau.\n" + assert "rede.glossar.txt (Video-Glossar, automatisch)" in capsys.readouterr().out + + +def test_main_video_glossary_given_explicitly_counts_once(tmp_path, monkeypatch, capsys): + folder = video_folder(tmp_path) + shared = tmp_path / "gemeinsam.glossar.txt" + shared.write_text("Moskau\n", encoding="utf-8") + monkeypatch.setattr(ak, "correct_chunk", identity_correct) + monkeypatch.setattr(ak, "find_paragraph_starts", every_nth_start(5)) + run_main(monkeypatch, "-i", folder / "rede.txt", "-g", shared, "-g", folder / "rede.glossar.txt") + + out = capsys.readouterr().out + assert out.count("rede.glossar.txt") == 1 + assert "automatisch" not in out + # Shared glossaries first, as given + assert out.index("gemeinsam.glossar.txt") < out.index("rede.glossar.txt") + + +def test_with_video_glossary(tmp_path): + folder = video_folder(tmp_path) + shared = tmp_path / "gemeinsam.glossar.txt" + assert ak.with_video_glossary([shared], folder) == ( + [shared, folder / "rede.glossar.txt"], folder / "rede.glossar.txt" + ) + # Without a glossary named like the folder nothing is added + assert ak.with_video_glossary([shared], tmp_path) == ([shared], None) def test_main_url_output_option_and_kept_raw_text(from_url, tmp_path, monkeypatch, capsys): @@ -762,19 +812,21 @@ def test_cli_errors(sample_copy, monkeypatch, capsys, args, message): def test_main_url_time_range_default_names(from_url, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) + work = tmp_path / "abschriften" / "Kanal_Rede_2026-03-28" run_main(monkeypatch, "-u", URL, "-z", "0:00:02") name = "Kanal_Rede_2026-03-28_ab_00-00-02" - raw = (tmp_path / f"{name}.txt").read_text(encoding="utf-8") + raw = (work / f"{name}.txt").read_text(encoding="utf-8") assert raw == "[00:00:02 - 00:00:03]\nDas ist bekannt.\n" - result = (tmp_path / f"{name}.korrigiert.txt").read_text(encoding="utf-8") + result = (work / f"{name}.korrigiert.txt").read_text(encoding="utf-8") assert result == "[00:00:02 - 00:00:03]\n\nDas ist bekannt.\n" def test_main_url_range_from_beginning(from_url, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) + work = tmp_path / "abschriften" / "Kanal_Rede_2026-03-28" run_main(monkeypatch, "-u", URL, "-z", "-0:00:02") - raw = (tmp_path / "Kanal_Rede_2026-03-28_bis_00-00-02.txt").read_text(encoding="utf-8") + raw = (work / "Kanal_Rede_2026-03-28_bis_00-00-02.txt").read_text(encoding="utf-8") assert raw == "[00:00:01 - 00:00:02]\nDer Kremmel ist in Moskau.\n" @@ -926,14 +978,15 @@ def test_main_url_several_ranges(fake_yt_dlp, yt_dlp_installed, tmp_path, monkey monkeypatch.setattr(subprocess, "run", fake_yt_dlp(VIDEO, vtt=MULTI_VTT)) sent = chunk_recorder(monkeypatch) monkeypatch.chdir(tmp_path) + work = tmp_path / "abschriften" / "Kanal_Rede_2026-03-28" run_main(monkeypatch, "-u", URL, "-z", "0:00:00-0:00:30 0:02", "-g", kreml_glossary(tmp_path)) name = "Kanal_Rede_2026-03-28_bis_00-00-30+ab_00-02-00" - assert (tmp_path / f"{name}.txt").read_text(encoding="utf-8") == ( + assert (work / f"{name}.txt").read_text(encoding="utf-8") == ( "[00:00:01 - 00:00:02]\nDer Kremmel ist in Moskau.\n\n" "[00:02:00 - 00:02:01]\nDas ist bekannt.\n" ) - assert (tmp_path / f"{name}.korrigiert.txt").read_text(encoding="utf-8") == ( + assert (work / f"{name}.korrigiert.txt").read_text(encoding="utf-8") == ( "[00:00:01 - 00:00:02]\n\nDer Kreml ist in Moskau.\n\n" "[00:02:00 - 00:02:01]\n\nDas ist bekannt.\n" ) diff --git a/tests/test_aenderungen.py b/tests/test_aenderungen.py index ce492dd..700c610 100644 --- a/tests/test_aenderungen.py +++ b/tests/test_aenderungen.py @@ -321,3 +321,19 @@ def test_report_and_summary_put_word_forms_last(tmp_path, capsys): out = capsys.readouterr().out.split("Zu prüfen: ")[1] assert out.startswith("2 Stellen (davon 1 Wortformen)") assert out.index("Professorin") < out.index("„das“ → „dass“") + + +def test_cli_uses_video_glossary(tmp_path, monkeypatch, capsys): + folder = tmp_path / "rede" + folder.mkdir() + raw = folder / "rede.txt" + raw.write_text("das Kremmel und\nBar heute\n", encoding="utf-8") + corrected = folder / "rede.korrigiert.txt" + corrected.write_text("Das Kreml und Bahr heute.\n", encoding="utf-8") + (folder / "rede.glossar.txt").write_text("Kremmel => Kreml\nEgon Bahr\n", encoding="utf-8") + + monkeypatch.setattr(sys, "argv", ["aenderungen.py", "-a", str(raw), "-b", str(corrected)]) + ae.main() + out = capsys.readouterr().out + assert "Video-Glossar (automatisch)" in out + assert "Zu prüfen: 0 Stellen" in out diff --git a/tests/test_extract_text.py b/tests/test_extract_text.py index 07131b9..404b1e7 100644 --- a/tests/test_extract_text.py +++ b/tests/test_extract_text.py @@ -1,4 +1,5 @@ """Tests for extract_text.py; yt-dlp is always replaced by fakes (no network).""" +import json import sys import pytest @@ -38,14 +39,19 @@ def test_default_language_is_original_audio(fake_yt_dlp, tmp_path, monkeypatch, monkeypatch.chdir(tmp_path) run_main(monkeypatch, "-u", URL) - text = (tmp_path / "Kanal_Eine_Rede_2026-03-28.txt").read_text(encoding="utf-8") + work = tmp_path / "abschriften" / "Kanal_Eine_Rede_2026-03-28" + text = (work / "Kanal_Eine_Rede_2026-03-28.txt").read_text(encoding="utf-8") assert text == "Guten Abend.\nSchön, dass Sie da sind.\n" assert "de-orig" in run.calls[1] assert "--write-auto-subs" in run.calls[1] out = capsys.readouterr().out assert "Kanal_Eine_Rede_2026-03-28.txt (2 Zeilen, 7 Wörter)" in out - # Only the text file remains, no VTT - assert [p.name for p in tmp_path.iterdir()] == ["Kanal_Eine_Rede_2026-03-28.txt"] + # Only the text file and where it comes from remain, no VTT + assert sorted(p.name for p in work.iterdir()) == [ + "Kanal_Eine_Rede_2026-03-28.quelle.json", "Kanal_Eine_Rede_2026-03-28.txt", + ] + source = json.loads((work / "Kanal_Eine_Rede_2026-03-28.quelle.json").read_text()) + assert source["kanal"] == "Kanal" and source["url"].endswith("v=abc123") def test_default_language_ignores_dubbed_tracks(fake_yt_dlp, tmp_path, monkeypatch): @@ -133,7 +139,8 @@ def test_time_range_default_name(fake_yt_dlp, tmp_path, monkeypatch): run_main(monkeypatch, "-u", URL, "-z", "0:00:02-0:00:03") name = "Kanal_Eine_Rede_2026-03-28_00-00-02_bis_00-00-03.txt" - assert (tmp_path / name).read_text(encoding="utf-8") == ( + work = tmp_path / "abschriften" / "Kanal_Eine_Rede_2026-03-28" + assert (work / name).read_text(encoding="utf-8") == ( "[00:00:02 - 00:00:03]\nSchön, dass Sie da sind.\n" ) @@ -183,7 +190,8 @@ def test_several_ranges_flow_text_keeps_headers(fake_yt_dlp, tmp_path, monkeypat run_main(monkeypatch, "-u", URL, "-z", "0:00:00-0:00:30 0:02", "-f") name = "Kanal_Eine_Rede_2026-03-28_bis_00-00-30+ab_00-02-00.txt" - assert (tmp_path / name).read_text(encoding="utf-8") == ( + work = tmp_path / "abschriften" / "Kanal_Eine_Rede_2026-03-28" + assert (work / name).read_text(encoding="utf-8") == ( "[00:00:01 - 00:00:02]\nErster Satz der\nRede.\n\n" "[00:02:00 - 00:02:01]\nDritter Satz, der\netwas länger ist.\n" ) diff --git a/tests/test_glossar_vorschlag.py b/tests/test_glossar_vorschlag.py index ac86ec2..0e3913e 100644 --- a/tests/test_glossar_vorschlag.py +++ b/tests/test_glossar_vorschlag.py @@ -160,3 +160,18 @@ def test_time_headers_never_reach_model_or_changes(tmp_path, monkeypatch): draft = out.read_text(encoding="utf-8") assert "# ? Kremmel => Kreml (1×)" in draft assert "00:0" not in draft.replace(str(raw), "") + + +def test_main_uses_video_glossary(tmp_path, monkeypatch, capsys): + folder = tmp_path / "rede" + folder.mkdir() + raw = folder / "rede.txt" + raw.write_text(RAW + "\n", encoding="utf-8") + (folder / "rede.glossar.txt").write_text("Kremmel => Kreml\n", encoding="utf-8") + model = fake_model(ENTRIES) + monkeypatch.setattr(urllib.request, "urlopen", model) + run_main(monkeypatch, "-i", raw) + + sent = model.calls[0]["messages"][1]["content"] + assert "Kreml" in sent and "Kremmel" not in sent + assert "Video-Glossar (automatisch)" in capsys.readouterr().out diff --git a/tests/test_yt_untertitel.py b/tests/test_yt_untertitel.py index 7bd446e..2049e78 100644 --- a/tests/test_yt_untertitel.py +++ b/tests/test_yt_untertitel.py @@ -140,8 +140,11 @@ def test_yt_dlp_runs_in_temporary_directory(fake_yt_dlp, tmp_path, monkeypatch): assert all(cwd is not None and Path(cwd) != tmp_path for cwd in run.cwds) template = run.calls[1][run.calls[1].index("-o") + 1] - assert template == f"{tmp_path / 'Rede'}.%(ext)s" - assert (tmp_path / "Rede.de-orig.vtt").is_file() + # Without -d: working folder abschriften//, named after -n + work = tmp_path / "abschriften" / "Rede" + assert template == f"{work / 'Rede'}.%(ext)s" + assert (work / "Rede.de-orig.vtt").is_file() + assert (work / "Rede.quelle.json").is_file() def test_main_prefers_manual_and_default_name(fake_yt_dlp, tmp_path, monkeypatch): diff --git a/yt_untertitel.py b/yt_untertitel.py index 62b99bd..94c1987 100755 --- a/yt_untertitel.py +++ b/yt_untertitel.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Lädt die Untertitel eines YouTube-Videos als VTT-Datei herunter (über yt-dlp), als Eingabe für vtt2txt.py bzw. abschrift_korrigieren.py.""" +from datetime import date from pathlib import Path import argparse import json @@ -14,6 +15,7 @@ YT_DLP = "yt-dlp" NAME_WORD = re.compile(r"\w+(?:[-.]\w+)*") # keeps "Krone-Schmalz", "28.03.2026" MAX_CHANNEL_CHARS = 40 MAX_TITLE_CHARS = 60 +WORK_DIR = Path("abschriften") # ein Arbeitsordner je Video: abschriften// def run_yt_dlp(arguments): @@ -112,6 +114,31 @@ def default_name(info): return "_".join(p for p in parts if p) or info.get("id") or "video" +def work_folder(name): + """Working folder of one video: abschriften//""" + return WORK_DIR / name + + +def write_source(info, path): + """Store where a transcript comes from (.quelle.json), for the + header of the finished result.""" + url = info.get("webpage_url") or ( + f"https://www.youtube.com/watch?v={info['id']}" if info.get("id") else None + ) + source = { + "titel": info.get("title"), + "kanal": info.get("channel") or info.get("uploader"), + "datum": info.get("upload_date"), + "url": url, + "id": info.get("id"), + "dauer": info.get("duration"), + "abgerufen": date.today().isoformat(), + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(source, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return path + + def available_tracks(info): """Manual tracks and the automatic ones usable for this program.""" manual = sorted((info.get("subtitles") or {}).keys()) @@ -164,8 +191,8 @@ def parse_args(): "-l", "--lang", default="de", help="Sprache der Untertitel (Standard: de)" ) parser.add_argument( - "-d", "--dir", type=Path, default=Path("."), - help="Zielverzeichnis (Standard: aktuelles Verzeichnis)", + "-d", "--dir", type=Path, + help="Zielverzeichnis (Standard: Arbeitsordner abschriften//)", ) args = parser.parse_args() @@ -189,9 +216,12 @@ def main(): kind = "automatische" if is_auto else "manuelle" print(f"Lade {kind} Untertitel „{track}“ …", flush=True) - args.dir.mkdir(parents=True, exist_ok=True) - target_base = args.dir / (args.name or default_name(info)) + name = args.name or default_name(info) + folder = args.dir or work_folder(name) + folder.mkdir(parents=True, exist_ok=True) + target_base = folder / name download(args.url, track, is_auto, target_base) + write_source(info, folder / f"{name}.quelle.json") target = Path(f"{target_base}.{track}.vtt") if not target.is_file():