diff --git a/README.md b/README.md index 3003bcb..a6c2983 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ Die Werkzeuge sind einzeln oder hintereinander nutzbar: - für den Download: [yt-dlp](https://github.com/yt-dlp/yt-dlp) (`pip install yt-dlp`) - Ollama auf `127.0.0.1:11434` mit dem Modell `qwen3:32b` (`ollama pull qwen3:32b`) + `rede_korrigieren.py` und `glossar_vorschlag.py` prüfen beides vor dem Start und + brechen sonst mit einem Hinweis ab (`sudo systemctl start ollama` bzw. `ollama pull …`) Für gute Geschwindigkeit muss das Modell komplett in den Grafikspeicher passen. Mit `OLLAMA_NUM_PARALLEL=1` braucht `qwen3:32b` bei 16k Kontext rund 24 GB diff --git a/glossar_vorschlag.py b/glossar_vorschlag.py index 2847cf7..ecaa520 100755 --- a/glossar_vorschlag.py +++ b/glossar_vorschlag.py @@ -9,11 +9,11 @@ import hashlib import io import json import re -import urllib.request from aenderungen import compare, read_raw from rede_korrigieren import ( - MODEL, apply_replacements, chat_request, read_glossary, split_chunks, + MODEL, apply_replacements, chat_request, check_ollama, ollama_open, read_glossary, + split_chunks, ) SYSTEM = """Du suchst Hörfehler in der automatischen Abschrift einer deutschen Rede. @@ -77,7 +77,7 @@ def suggest_chunk(chunk, model): ], "options": {"temperature": 0, "num_ctx": 16384, "num_predict": 4096}, } - with urllib.request.urlopen(chat_request(payload), timeout=600) as response: + with ollama_open(chat_request(payload)) as response: result = json.load(response) try: entries = json.loads(result["message"]["content"])["eintraege"] @@ -227,6 +227,7 @@ def main(): with redirect_stdout(io.StringIO()): text = apply_replacements(text, replacements) + check_ollama(args.model) cache_dir = args.output.with_suffix(".parts") cache_dir.mkdir(parents=True, exist_ok=True) suggestions = model_suggestions(text, args.model, cache_dir, known) diff --git a/pytest.ini b/pytest.ini index a1ea7e7..98641db 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,3 +3,5 @@ testpaths = tests pythonpath = . # Project has no async code; skip the globally installed asyncio plugin addopts = -p no:asyncio +markers = + real_ollama_check: run the real Ollama server check (urlopen still faked) diff --git a/rede_korrigieren.py b/rede_korrigieren.py index e0086d4..2c6c19f 100755 --- a/rede_korrigieren.py +++ b/rede_korrigieren.py @@ -8,6 +8,7 @@ import re import shutil import sys import time +import urllib.error import urllib.request from aenderungen import REVIEW_MARK, print_summary, report_path, write_report @@ -17,6 +18,7 @@ from yt_untertitel import YT_DLP, base_language, default_name, original_language MODEL = "qwen3:32b" OLLAMA_URL = "http://127.0.0.1:11434/api/chat" +OLLAMA_TAGS_URL = "http://127.0.0.1:11434/api/tags" CHUNK_SIZE = 1200 # Zielgröße eines Abschnitts in Wörtern MAX_EXTRA_WORDS = 300 # so weit darf ein Abschnitt bis zum nächsten Satzende wachsen @@ -237,6 +239,31 @@ def chat_request(payload): ) +def ollama_open(request, timeout=600): + """urlopen() for Ollama; a missing server ends the program with a short message.""" + try: + return urllib.request.urlopen(request, timeout=timeout) + except urllib.error.HTTPError: + raise + except urllib.error.URLError as error: + raise SystemExit( + f"Ollama nicht erreichbar ({error.reason}). Läuft der Dienst? " + "Starten mit: sudo systemctl start ollama (oder: ollama serve)" + ) from None + + +def check_ollama(model): + """Stop early, before any download, if Ollama is down or lacks the model.""" + with ollama_open(OLLAMA_TAGS_URL, timeout=10) as response: + installed = {m.get("name") for m in json.load(response).get("models", [])} + wanted = model if ":" in model else f"{model}:latest" + if wanted not in installed: + raise SystemExit( + f"Modell „{model}“ ist in Ollama nicht installiert. " + f"Installieren mit: ollama pull {model}" + ) + + def correct_chunk(chunk, index, system): payload = { "model": MODEL, @@ -263,7 +290,7 @@ def correct_chunk(chunk, index, system): words = 0 result = {} - with urllib.request.urlopen(request, timeout=600) as response: + with ollama_open(request) as response: for line in response: if not line.strip(): continue @@ -328,7 +355,7 @@ def find_paragraph_starts(sentences, first, system): "num_predict": 2048, }, } - with urllib.request.urlopen(chat_request(payload), timeout=600) as response: + with ollama_open(chat_request(payload)) as response: result = json.load(response) content = result["message"]["content"] @@ -542,6 +569,7 @@ def main(): # reuses cached parts of another MODEL = args.model print(f"Modell: {MODEL}") + check_ollama(MODEL) raw, target, lang = read_source(args) print(f"Sprache der Rede: {lang}") parts_dir = target.with_suffix(".parts") diff --git a/tests/conftest.py b/tests/conftest.py index 31250c6..db2f92f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -"""Shared fixtures: yt-dlp is always replaced by fakes (no network).""" +"""Shared fixtures: yt-dlp and Ollama are always replaced by fakes (no network).""" import json import shutil import subprocess @@ -7,6 +7,17 @@ from pathlib import Path import pytest +@pytest.fixture(autouse=True) +def ollama_available(request, monkeypatch): + """Skip the Ollama server check; tests marked real_ollama_check test it.""" + if "real_ollama_check" in request.keywords: + return + import glossar_vorschlag + import rede_korrigieren + for module in (rede_korrigieren, glossar_vorschlag): + monkeypatch.setattr(module, "check_ollama", lambda model: None) + + @pytest.fixture def fake_yt_dlp(): """Factory for a subprocess.run stand-in. diff --git a/tests/test_glossar_vorschlag.py b/tests/test_glossar_vorschlag.py index f504896..e5a8fde 100644 --- a/tests/test_glossar_vorschlag.py +++ b/tests/test_glossar_vorschlag.py @@ -2,6 +2,7 @@ import io import json import sys +import urllib.request import pytest @@ -63,7 +64,7 @@ def run_main(monkeypatch, *args): def test_main_filters_and_formats(raw_file, glossary, monkeypatch, capsys): model = fake_model(ENTRIES) - monkeypatch.setattr(gv.urllib.request, "urlopen", model) + monkeypatch.setattr(urllib.request, "urlopen", model) run_main(monkeypatch, "-i", raw_file, "-g", glossary, "-m", "test:1b") draft = raw_file.with_name("rede.glossar.entwurf.txt").read_text(encoding="utf-8") @@ -84,12 +85,12 @@ def test_main_filters_and_formats(raw_file, glossary, monkeypatch, capsys): assert "Vermutete Hörfehler: 2 (1 sicher, 1 unsicher)" in out # Second run uses the cache - monkeypatch.setattr(gv.urllib.request, "urlopen", no_model_calls) + monkeypatch.setattr(urllib.request, "urlopen", no_model_calls) run_main(monkeypatch, "-i", raw_file, "-g", glossary, "-m", "test:1b") def test_correction_changes_are_commented_out(raw_file, glossary, tmp_path, monkeypatch): - monkeypatch.setattr(gv.urllib.request, "urlopen", fake_model(ENTRIES[:2])) + monkeypatch.setattr(urllib.request, "urlopen", fake_model(ENTRIES[:2])) corrected = tmp_path / "rede.korrigiert.txt" # Corrected without glossary: "Kremmel" became "Kremml" (a model error) corrected.write_text( @@ -110,7 +111,7 @@ def test_correction_changes_are_commented_out(raw_file, glossary, tmp_path, monk def test_drafted_entries_are_valid_glossary(raw_file, monkeypatch, tmp_path): - monkeypatch.setattr(gv.urllib.request, "urlopen", fake_model(ENTRIES[:2])) + monkeypatch.setattr(urllib.request, "urlopen", fake_model(ENTRIES[:2])) out = tmp_path / "entwurf.txt" run_main(monkeypatch, "-i", raw_file, "-o", out) assert rk.read_glossary(out) == ([], []) # nothing active @@ -126,7 +127,7 @@ def test_unreadable_model_answer(raw_file, monkeypatch, tmp_path, capsys): def urlopen(request, timeout=None): return io.BytesIO(json.dumps({"message": {"content": "kein JSON"}}).encode()) - monkeypatch.setattr(gv.urllib.request, "urlopen", urlopen) + monkeypatch.setattr(urllib.request, "urlopen", urlopen) run_main(monkeypatch, "-i", raw_file, "-o", tmp_path / "e.txt") assert "unlesbar" in capsys.readouterr().out diff --git a/tests/test_rede_korrigieren.py b/tests/test_rede_korrigieren.py index 182c168..33ff12f 100644 --- a/tests/test_rede_korrigieren.py +++ b/tests/test_rede_korrigieren.py @@ -566,3 +566,52 @@ def test_cli_time_errors(sample_copy, monkeypatch, capsys, args, message): run_main(monkeypatch, *args) assert exc.value.code == 2 assert message in capsys.readouterr().err + + +# === Ollama availability === + +def tags_urlopen(*names): + body = json.dumps({"models": [{"name": n} for n in names]}).encode("utf-8") + return fake_urlopen(body) + + +@pytest.mark.real_ollama_check +@pytest.mark.parametrize("model", ["qwen3:32b", "llama3"]) +def test_check_ollama_model_installed(monkeypatch, model): + monkeypatch.setattr(rk.urllib.request, "urlopen", tags_urlopen("qwen3:32b", "llama3:latest")) + rk.check_ollama(model) + + +@pytest.mark.real_ollama_check +def test_check_ollama_model_missing(monkeypatch): + monkeypatch.setattr(rk.urllib.request, "urlopen", tags_urlopen("qwen3:8b")) + with pytest.raises(SystemExit, match="ollama pull qwen3:32b"): + rk.check_ollama("qwen3:32b") + + +def refused(request, timeout=None): + raise rk.urllib.error.URLError(ConnectionRefusedError(111, "Connection refused")) + + +@pytest.mark.real_ollama_check +def test_main_ollama_down_stops_before_download(fake_yt_dlp, yt_dlp_installed, monkeypatch): + run = fake_yt_dlp(VIDEO, vtt=VTT) + monkeypatch.setattr(subprocess, "run", run) + monkeypatch.setattr(rk.urllib.request, "urlopen", refused) + with pytest.raises(SystemExit, match="Ollama nicht erreichbar.*systemctl start ollama"): + run_main(monkeypatch, "-u", URL) + assert run.calls == [] + + +def test_correct_chunk_ollama_gone_midway(monkeypatch): + monkeypatch.setattr(rk.urllib.request, "urlopen", refused) + with pytest.raises(SystemExit, match="Ollama nicht erreichbar"): + rk.correct_chunk("Text.", 1, "system") + + +def test_ollama_open_passes_http_errors(monkeypatch): + def not_found(request, timeout=None): + raise rk.urllib.error.HTTPError(rk.OLLAMA_URL, 404, "Not Found", {}, None) + monkeypatch.setattr(rk.urllib.request, "urlopen", not_found) + with pytest.raises(rk.urllib.error.HTTPError): + rk.ollama_open(rk.OLLAMA_URL)