feat: rede_korrigieren.py -u URL – von YouTube direkt zum korrigierten Text
- extract_text.fetch_text() aus main() herausgelöst und wiederverwendet - -u/--url statt -i, -l/--lang für die Sprache (Standard: Original-Audio) - Ohne -o: <Video-ID>.korrigiert.txt; der Rohtext wird daneben gespeichert, eine vorhandene, abweichende Rohtext-Datei aber nie überschrieben Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
bc09ddf3bf
commit
5a072deca3
4 changed files with 133 additions and 18 deletions
16
README.md
16
README.md
|
|
@ -29,7 +29,14 @@ der Lauf wird etwa 12-mal langsamer.
|
||||||
|
|
||||||
## Aufruf
|
## Aufruf
|
||||||
|
|
||||||
Von der YouTube-URL zum korrigierten Text:
|
Von der YouTube-URL zum korrigierten Text in einem Befehl:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./rede_korrigieren.py -u 'https://www.youtube.com/watch?v=rm6lADgf6GQ' -o lahnstein.korrigiert.txt
|
||||||
|
# → lahnstein.korrigiert.txt, lahnstein.txt (Rohtext), lahnstein.aenderungen.html
|
||||||
|
```
|
||||||
|
|
||||||
|
Oder in zwei Schritten, um den Rohtext vorher anzusehen:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./extract_text.py -u 'https://www.youtube.com/watch?v=rm6lADgf6GQ' -o lahnstein.txt
|
./extract_text.py -u 'https://www.youtube.com/watch?v=rm6lADgf6GQ' -o lahnstein.txt
|
||||||
|
|
@ -126,12 +133,15 @@ werden trotzdem umgewandelt; der Exit-Code ist dann 1.
|
||||||
./rede_korrigieren.py -i rede.txt -g rede.glossar.txt # mit Glossar
|
./rede_korrigieren.py -i rede.txt -g rede.glossar.txt # mit Glossar
|
||||||
./rede_korrigieren.py -i rede.txt -g allgemein.glossar.txt -g rede.glossar.txt # kombiniert
|
./rede_korrigieren.py -i rede.txt -g allgemein.glossar.txt -g rede.glossar.txt # kombiniert
|
||||||
./rede_korrigieren.py -i rede.de.vtt # VTT direkt → rede.de.korrigiert.txt
|
./rede_korrigieren.py -i rede.de.vtt # VTT direkt → rede.de.korrigiert.txt
|
||||||
|
./rede_korrigieren.py -u URL # YouTube → <Video-ID>.korrigiert.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
| Option | Bedeutung |
|
| Option | Bedeutung |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `-i`, `--input` | Eingabedatei: Rohabschrift (`.txt`) oder Untertitel (`.vtt`, wird wie mit `vtt2txt.py` umgewandelt), Pflicht |
|
| `-i`, `--input` | Eingabedatei: Rohabschrift (`.txt`) oder Untertitel (`.vtt`, wird wie mit `vtt2txt.py` umgewandelt) |
|
||||||
| `-o`, `--output` | Ausgabedatei; Standard: `<Eingabe ohne Endung>.korrigiert.txt` neben der Eingabe |
|
| `-u`, `--url` | statt `-i`: YouTube-URL; Untertitel wie bei `extract_text.py`. Der Rohtext wird neben der Ausgabe gespeichert (`rede.korrigiert.txt` → `rede.txt`, sonst `<Name>.roh.txt`), eine vorhandene, abweichende Datei aber nie überschrieben |
|
||||||
|
| `-l`, `--lang` | nur mit `-u`: Sprache; Standard: Sprache des Original-Audios |
|
||||||
|
| `-o`, `--output` | Ausgabedatei; Standard: `<Eingabe ohne Endung>.korrigiert.txt` neben der Eingabe, mit `-u` `<Video-ID>.korrigiert.txt` |
|
||||||
| `-g`, `--glossar` | Glossar mit festen Ersetzungen und Schreibweisen; mehrfach angeben zum Kombinieren (Ersetzungen in Reihenfolge der Angabe) |
|
| `-g`, `--glossar` | Glossar mit festen Ersetzungen und Schreibweisen; mehrfach angeben zum Kombinieren (Ersetzungen in Reihenfolge der Angabe) |
|
||||||
| `-m`, `--model` | Ollama-Modell; Standard: `qwen3:32b` (Konstante `MODEL`) |
|
| `-m`, `--model` | Ollama-Modell; Standard: `qwen3:32b` (Konstante `MODEL`) |
|
||||||
| `--kein-bericht` | keinen Änderungsbericht schreiben |
|
| `--kein-bericht` | keinen Änderungsbericht schreiben |
|
||||||
|
|
|
||||||
|
|
@ -41,13 +41,15 @@ def parse_args():
|
||||||
return args
|
return args
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def fetch_text(url, lang=None):
|
||||||
args = parse_args()
|
"""Download the subtitles of a video; return (video info, text lines).
|
||||||
|
|
||||||
info = fetch_info(args.url)
|
Without `lang` the language of the original audio is used.
|
||||||
|
"""
|
||||||
|
info = fetch_info(url)
|
||||||
print(f"Video: {info.get('title', '?')}")
|
print(f"Video: {info.get('title', '?')}")
|
||||||
|
|
||||||
lang = args.lang or original_language(info)
|
lang = lang or original_language(info)
|
||||||
chosen = choose_track(info, lang) if lang else None
|
chosen = choose_track(info, lang) if lang else None
|
||||||
if chosen is None:
|
if chosen is None:
|
||||||
raise SystemExit(no_track_message(info, lang))
|
raise SystemExit(no_track_message(info, lang))
|
||||||
|
|
@ -58,7 +60,7 @@ def main():
|
||||||
# The VTT file is only an intermediate step
|
# The VTT file is only an intermediate step
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
target_base = Path(tmp) / "untertitel"
|
target_base = Path(tmp) / "untertitel"
|
||||||
download(args.url, track, is_auto, target_base)
|
download(url, track, is_auto, target_base)
|
||||||
vtt = Path(f"{target_base}.{track}.vtt")
|
vtt = Path(f"{target_base}.{track}.vtt")
|
||||||
if not vtt.is_file():
|
if not vtt.is_file():
|
||||||
raise SystemExit("yt-dlp hat keine Untertitel-Datei geschrieben.")
|
raise SystemExit("yt-dlp hat keine Untertitel-Datei geschrieben.")
|
||||||
|
|
@ -66,6 +68,12 @@ def main():
|
||||||
|
|
||||||
if not lines:
|
if not lines:
|
||||||
raise SystemExit("Die Untertitel enthalten keinen Text.")
|
raise SystemExit("Die Untertitel enthalten keinen Text.")
|
||||||
|
return info, lines
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
_, lines = fetch_text(args.url, args.lang)
|
||||||
|
|
||||||
if args.fliesstext:
|
if args.fliesstext:
|
||||||
# Never split words, not even at hyphens ("EU-Gipfel")
|
# Never split words, not even at hyphens ("EU-Gipfel")
|
||||||
|
|
|
||||||
|
|
@ -5,12 +5,15 @@ import argparse
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
|
||||||
from aenderungen import REVIEW_MARK, print_summary, report_path, write_report
|
from aenderungen import REVIEW_MARK, print_summary, report_path, write_report
|
||||||
|
from extract_text import fetch_text
|
||||||
from vtt2txt import vtt_to_lines
|
from vtt2txt import vtt_to_lines
|
||||||
|
from yt_untertitel import YT_DLP
|
||||||
|
|
||||||
MODEL = "qwen3:32b"
|
MODEL = "qwen3:32b"
|
||||||
OLLAMA_URL = "http://127.0.0.1:11434/api/chat"
|
OLLAMA_URL = "http://127.0.0.1:11434/api/chat"
|
||||||
|
|
@ -362,13 +365,23 @@ def parse_args():
|
||||||
description="Korrigiert eine automatische Abschrift einer deutschen Rede "
|
description="Korrigiert eine automatische Abschrift einer deutschen Rede "
|
||||||
"mit einem lokalen Sprachmodell über Ollama."
|
"mit einem lokalen Sprachmodell über Ollama."
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
source = parser.add_mutually_exclusive_group(required=True)
|
||||||
"-i", "--input", required=True, type=Path,
|
source.add_argument(
|
||||||
|
"-i", "--input", type=Path,
|
||||||
help="Eingabedatei: Rohabschrift (.txt) oder Untertitel (.vtt)",
|
help="Eingabedatei: Rohabschrift (.txt) oder Untertitel (.vtt)",
|
||||||
)
|
)
|
||||||
|
source.add_argument(
|
||||||
|
"-u", "--url",
|
||||||
|
help="YouTube-URL: Untertitel laden (Rohtext wird neben der Ausgabe gespeichert)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"-l", "--lang",
|
||||||
|
help="nur mit -u: Sprache der Untertitel (Standard: Sprache des Original-Audios)",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"-o", "--output", type=Path,
|
"-o", "--output", type=Path,
|
||||||
help="Ausgabedatei (Standard: <Eingabe ohne Endung>.korrigiert.txt)",
|
help="Ausgabedatei (Standard: <Eingabe ohne Endung>.korrigiert.txt, "
|
||||||
|
"mit -u <Video-ID>.korrigiert.txt)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"-g", "--glossar", type=Path, action="append", default=[],
|
"-g", "--glossar", type=Path, action="append", default=[],
|
||||||
|
|
@ -384,11 +397,19 @@ def parse_args():
|
||||||
)
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if not args.input.is_file():
|
|
||||||
parser.error(f"Eingabedatei nicht gefunden: {args.input}")
|
|
||||||
for glossary in args.glossar:
|
for glossary in args.glossar:
|
||||||
if not glossary.is_file():
|
if not glossary.is_file():
|
||||||
parser.error(f"Glossar nicht gefunden: {glossary}")
|
parser.error(f"Glossar nicht gefunden: {glossary}")
|
||||||
|
if args.url is not None:
|
||||||
|
if shutil.which(YT_DLP) is None:
|
||||||
|
parser.error(f"{YT_DLP} nicht gefunden (Installation: pip install yt-dlp).")
|
||||||
|
# Without -o the name depends on the video ID, known after the download
|
||||||
|
return args
|
||||||
|
|
||||||
|
if args.lang is not None:
|
||||||
|
parser.error("-l/--lang gibt es nur zusammen mit -u/--url.")
|
||||||
|
if not args.input.is_file():
|
||||||
|
parser.error(f"Eingabedatei nicht gefunden: {args.input}")
|
||||||
if args.output is None:
|
if args.output is None:
|
||||||
args.output = args.input.with_name(f"{args.input.stem}.korrigiert.txt")
|
args.output = args.input.with_name(f"{args.input.stem}.korrigiert.txt")
|
||||||
if args.output.resolve() == args.input.resolve():
|
if args.output.resolve() == args.input.resolve():
|
||||||
|
|
@ -396,6 +417,38 @@ def parse_args():
|
||||||
return args
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
def raw_text_path(target):
|
||||||
|
"""Where -u keeps the raw transcript: rede.korrigiert.txt -> rede.txt."""
|
||||||
|
if target.name.endswith(".korrigiert.txt"):
|
||||||
|
return target.with_name(target.name.removesuffix(".korrigiert.txt") + ".txt")
|
||||||
|
return target.with_name(f"{target.stem}.roh.txt")
|
||||||
|
|
||||||
|
|
||||||
|
def save_raw_text(path, raw):
|
||||||
|
"""Keep the downloaded transcript for glossary work, never overwrite edits."""
|
||||||
|
if path.exists() and path.read_text(encoding="utf-8") != raw:
|
||||||
|
print(f"WARNUNG: {path} existiert mit anderem Inhalt und bleibt unverändert.")
|
||||||
|
return
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(raw, encoding="utf-8")
|
||||||
|
print(f"Rohtext: {path}")
|
||||||
|
|
||||||
|
|
||||||
|
def read_source(args):
|
||||||
|
"""Return the raw transcript and the output path."""
|
||||||
|
if args.url is not None:
|
||||||
|
info, lines = fetch_text(args.url, args.lang)
|
||||||
|
target = args.output or Path(f"{info['id']}.korrigiert.txt")
|
||||||
|
raw = "\n".join(lines) + "\n"
|
||||||
|
save_raw_text(raw_text_path(target), raw)
|
||||||
|
return raw, target
|
||||||
|
|
||||||
|
raw = args.input.read_text(encoding="utf-8-sig")
|
||||||
|
if args.input.suffix.lower() == ".vtt":
|
||||||
|
raw = "\n".join(vtt_to_lines(raw))
|
||||||
|
return raw, args.output
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
global MODEL
|
global MODEL
|
||||||
args = parse_args()
|
args = parse_args()
|
||||||
|
|
@ -403,8 +456,7 @@ def main():
|
||||||
# reuses cached parts of another
|
# reuses cached parts of another
|
||||||
MODEL = args.model
|
MODEL = args.model
|
||||||
print(f"Modell: {MODEL}")
|
print(f"Modell: {MODEL}")
|
||||||
source = args.input
|
raw, target = read_source(args)
|
||||||
target = args.output
|
|
||||||
parts_dir = target.with_suffix(".parts")
|
parts_dir = target.with_suffix(".parts")
|
||||||
|
|
||||||
terms, replacements = [], []
|
terms, replacements = [], []
|
||||||
|
|
@ -418,9 +470,6 @@ def main():
|
||||||
replacements += more_replacements
|
replacements += more_replacements
|
||||||
system = build_system(terms)
|
system = build_system(terms)
|
||||||
|
|
||||||
raw = source.read_text(encoding="utf-8-sig")
|
|
||||||
if source.suffix.lower() == ".vtt":
|
|
||||||
raw = "\n".join(vtt_to_lines(raw))
|
|
||||||
# Subtitle line breaks carry no meaning; send plain running text
|
# Subtitle line breaks carry no meaning; send plain running text
|
||||||
text = " ".join(raw.split())
|
text = " ".join(raw.split())
|
||||||
# The change report compares against this, so glossary effects show up too
|
# The change report compares against this, so glossary effects show up too
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import shutil
|
import shutil
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
@ -375,6 +376,51 @@ def test_correct_chunk_sends_model(monkeypatch):
|
||||||
assert sent["model"] == "anderes:7b"
|
assert sent["model"] == "anderes:7b"
|
||||||
|
|
||||||
|
|
||||||
|
URL = "https://www.youtube.com/watch?v=abc123"
|
||||||
|
VIDEO = {"id": "abc123", "title": "Rede", "subtitles": {}, "automatic_captions": {"de-orig": []}}
|
||||||
|
VTT = (
|
||||||
|
"WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nDer Kremmel ist in Moskau.\n\n"
|
||||||
|
"00:00:02.000 --> 00:00:03.000\nDer Kremmel ist in Moskau.\nDas ist bekannt.\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def from_url(fake_yt_dlp, yt_dlp_installed, monkeypatch):
|
||||||
|
"""Fake download plus fake model for runs with -u."""
|
||||||
|
monkeypatch.setattr(subprocess, "run", fake_yt_dlp(VIDEO, vtt=VTT))
|
||||||
|
monkeypatch.setattr(rk, "correct_chunk", identity_correct)
|
||||||
|
monkeypatch.setattr(rk, "find_paragraph_starts", every_nth_start(5))
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_url_default_names(from_url, tmp_path, monkeypatch):
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
run_main(monkeypatch, "-u", URL, "-g", SHARED_GLOSSARY)
|
||||||
|
|
||||||
|
raw = (tmp_path / "abc123.txt").read_text(encoding="utf-8")
|
||||||
|
assert raw == "Der Kremmel ist in Moskau.\nDas ist bekannt.\n"
|
||||||
|
result = (tmp_path / "abc123.korrigiert.txt").read_text(encoding="utf-8")
|
||||||
|
assert result == "Der Kreml ist in Moskau. Das ist bekannt.\n"
|
||||||
|
assert (tmp_path / "abc123.aenderungen.html").is_file()
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_url_output_option_and_kept_raw_text(from_url, tmp_path, monkeypatch, capsys):
|
||||||
|
target = tmp_path / "out" / "rede.txt"
|
||||||
|
raw_path = tmp_path / "out" / "rede.roh.txt"
|
||||||
|
run_main(monkeypatch, "-u", URL, "-o", target)
|
||||||
|
assert raw_path.is_file() and target.is_file()
|
||||||
|
|
||||||
|
# A raw text edited by hand is never overwritten
|
||||||
|
raw_path.write_text("von Hand bearbeitet\n", encoding="utf-8")
|
||||||
|
run_main(monkeypatch, "-u", URL, "-o", target)
|
||||||
|
assert raw_path.read_text(encoding="utf-8") == "von Hand bearbeitet\n"
|
||||||
|
assert "bleibt unverändert" in capsys.readouterr().out
|
||||||
|
|
||||||
|
|
||||||
|
def test_raw_text_path():
|
||||||
|
assert rk.raw_text_path(Path("a/x.korrigiert.txt")) == Path("a/x.txt")
|
||||||
|
assert rk.raw_text_path(Path("a/x.txt")) == Path("a/x.roh.txt")
|
||||||
|
|
||||||
|
|
||||||
def test_main_warns_on_word_count_deviation(sample_copy, monkeypatch, capsys):
|
def test_main_warns_on_word_count_deviation(sample_copy, monkeypatch, capsys):
|
||||||
def shortening(chunk, index, system):
|
def shortening(chunk, index, system):
|
||||||
words = chunk.split()
|
words = chunk.split()
|
||||||
|
|
@ -401,6 +447,8 @@ def test_main_empty_input(tmp_path, monkeypatch):
|
||||||
(["-i", "{sample}", "-o", "{sample}"], "nicht die Eingabedatei"),
|
(["-i", "{sample}", "-o", "{sample}"], "nicht die Eingabedatei"),
|
||||||
(["-i", "{sample}", "-g", "gibtsnicht.txt"], "Glossar nicht gefunden"),
|
(["-i", "{sample}", "-g", "gibtsnicht.txt"], "Glossar nicht gefunden"),
|
||||||
(["-i", "{sample}", "-g", str(GLOSSARY), "-g", "fehlt.txt"], "Glossar nicht gefunden: fehlt.txt"),
|
(["-i", "{sample}", "-g", str(GLOSSARY), "-g", "fehlt.txt"], "Glossar nicht gefunden: fehlt.txt"),
|
||||||
|
(["-i", "{sample}", "-u", "https://x"], "not allowed with"),
|
||||||
|
(["-i", "{sample}", "-l", "de"], "nur zusammen mit -u"),
|
||||||
])
|
])
|
||||||
def test_cli_errors(sample_copy, monkeypatch, capsys, args, message):
|
def test_cli_errors(sample_copy, monkeypatch, capsys, args, message):
|
||||||
args = [a.format(sample=sample_copy) for a in args]
|
args = [a.format(sample=sample_copy) for a in args]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue