Add prune_podcast_data.py: clean up podcast data Open Notebook leaves behind

Deleting an episode does not delete its data. DELETE /api/podcasts/episodes/{id}
resolves episode.audio_file and unlinks that one MP3 — the enclosing
data/podcasts/episodes/<uuid>/ directory, holding clips/ (one MP3 per dialogue
segment), outline.json and transcript.json, is never touched. Failed and
/retry-replaced runs leak a directory as well. Nothing reaps any of it: this
deployment had 15 directories on disk against 4 episodes in the database.

That is an upstream gap (roughly a shutil.rmtree of audio_path.parent.parent in
that handler), not something configurable here. Deliberately not patched by
bind-mounting a modified router — that would fork app logic into a deployment
repo and rot silently against pull_policy: always. Local cleanup instead.

scripts/prune_podcast_data.py reconciles the episode list from the API against
the directories on disk and removes:
  - orphaned directories (no corresponding episode), and
  - clips/ of completed episodes, since the clips are intermediate output once
    the final MP3 exists.

Dry-run by default; --yes applies, --keep-clips restricts it to orphans.
Two guards, both tested: it aborts when any job is running/pending (a running
job has no audio_file yet, so its working directory is indistinguishable from an
orphan) and it skips directories touched within the last 60 minutes (--min-age).
It also refuses to act if the API is unreachable, rather than guessing.

Deletion runs inside the container (docker compose exec … rm -rf): the container
writes as root, so the host user cannot remove those directories — a plain
host-side rmtree fails with EPERM after the first directory.

Verified on this deployment: freed 13 MB (10 orphaned dirs + clips of 3 episodes,
49 MB -> 36 MB); all four surviving episodes still stream byte-identical MP3s
from /audio afterwards.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

scripts/prune_podcast_data.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""Raeumt Podcast-Daten auf, die Open Notebook selbst liegen laesst.
+
+Zwei Arten von Muell entstehen unter notebook_data/podcasts/episodes/<uuid>/:
+
+1. Verwaiste Episodenordner. Beim Loeschen einer Episode entfernt Open Notebook
+   nur die finale MP3 (api/routers/podcasts.py: audio_path.unlink()), nicht den
+   Ordner mit clips/, outline.json und transcript.json. Auch fehlgeschlagene und
+   per /retry ersetzte Laeufe lassen ihren Ordner zurueck.
+
+2. clips/ fertiger Episoden. Die Einzelclips sind nur Zwischenprodukt; sobald die
+   finale MP3 existiert, werden sie nicht mehr gebraucht.
+
+Standardmaessig Trockenlauf — es wird nur angezeigt, was wegfiele. Erst mit --yes
+wird geloescht.
+
+    ./scripts/prune_podcast_data.py            # anzeigen
+    ./scripts/prune_podcast_data.py --yes      # loeschen
+    ./scripts/prune_podcast_data.py --keep-clips --yes   # nur Verwaiste loeschen
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+API = "http://127.0.0.1:5055"
+REPO = Path(__file__).resolve().parent.parent
+EPISODES_DIR = REPO / "notebook_data" / "podcasts" / "episodes"
+
+# Pfade in der DB sind Container-Pfade (/app/data/...), auf dem Host liegen sie
+# unter notebook_data/ (Bind-Mount aus docker-compose.yml).
+CONTAINER_PREFIX = "/app/data/"
+HOST_PREFIX = REPO / "notebook_data"
+
+# Ein laufender Job hat noch keine audio_file, sein Ordner ist also nicht als
+# "in Benutzung" erkennbar. Frisch angefasste Ordner deshalb grundsaetzlich in
+# Ruhe lassen.
+DEFAULT_MIN_AGE_MIN = 60
+
+
+def die(msg: str) -> None:
+    print(f"FEHLER: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+def fetch_episodes() -> list[dict]:
+    try:
+        with urllib.request.urlopen(f"{API}/api/podcasts/episodes", timeout=10) as r:
+            return json.load(r)
+    except (urllib.error.URLError, TimeoutError) as e:
+        die(f"API unter {API} nicht erreichbar ({e}). Laeuft der Container? "
+            "Ohne die Episodenliste laesst sich nicht sicher unterscheiden, "
+            "welche Ordner noch gebraucht werden.")
+
+
+def to_host_path(audio_file: str) -> Path | None:
+    """Container-Pfad der DB -> Pfad auf dem Host."""
+    if audio_file.startswith("file://"):
+        audio_file = audio_file[len("file://"):]
+    if audio_file.startswith(CONTAINER_PREFIX):
+        return HOST_PREFIX / audio_file[len(CONTAINER_PREFIX):]
+    p = Path(audio_file)
+    return p if p.is_absolute() and p.exists() else None
+
+
+def to_container_path(host_path: Path) -> str:
+    """Pfad auf dem Host -> Container-Pfad (Umkehrung von to_host_path)."""
+    rel = host_path.resolve().relative_to(HOST_PREFIX.resolve())
+    return CONTAINER_PREFIX + str(rel)
+
+
+def delete_in_container(paths: list[Path]) -> None:
+    """Loescht im Container statt auf dem Host.
+
+    Der open_notebook-Container laeuft als root und legt die Episodenordner
+    entsprechend root:root an — der Host-User darf sie nicht entfernen. Statt
+    dafuer sudo zu verlangen, loeschen wir dort, wo die Rechte ohnehin stimmen.
+    """
+    targets = [to_container_path(p) for p in paths]
+    guard = CONTAINER_PREFIX + "podcasts/episodes/"
+    for t in targets:  # Sicherheitsnetz gegen Pfade ausserhalb der Episoden.
+        if not t.startswith(guard):
+            die(f"Abbruch: {t} liegt ausserhalb von {guard}")
+    subprocess.run(
+        ["docker", "compose", "exec", "-T", "open_notebook", "rm", "-rf", "--", *targets],
+        cwd=REPO, check=True, capture_output=True,
+    )
+
+
+def dir_size_mb(path: Path) -> float:
+    return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) / 1e6
+
+
+def age_minutes(path: Path) -> float:
+    newest = max((f.stat().st_mtime for f in path.rglob("*")), default=path.stat().st_mtime)
+    return (time.time() - newest) / 60
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--yes", action="store_true",
+                    help="wirklich loeschen (ohne dies nur Trockenlauf)")
+    ap.add_argument("--keep-clips", action="store_true",
+                    help="clips/ fertiger Episoden behalten, nur verwaiste Ordner loeschen")
+    ap.add_argument("--min-age", type=int, default=DEFAULT_MIN_AGE_MIN, metavar="MIN",
+                    help=f"Ordner juenger als MIN Minuten nie anfassen (Default: {DEFAULT_MIN_AGE_MIN})")
+    args = ap.parse_args()
+
+    if not EPISODES_DIR.is_dir():
+        print(f"Nichts zu tun: {EPISODES_DIR} existiert nicht.")
+        return 0
+
+    episodes = fetch_episodes()
+
+    # Laufende Jobs schreiben gerade in ihren Ordner, den wir nicht zuordnen
+    # koennen (noch keine audio_file). Dann lieber gar nichts loeschen.
+    running = [e for e in episodes if e.get("job_status") in ("running", "pending", "submitted")]
+    if running:
+        die(f"{len(running)} Podcast-Job(s) laufen gerade "
+            f"({', '.join(e.get('name') or e['id'] for e in running)}). "
+            "Bitte warten, bis sie fertig sind — sonst wuerde ihr Arbeitsverzeichnis geloescht.")
+
+    # Ordner, die zu einer existierenden Episode gehoeren.
+    live: dict[Path, dict] = {}
+    for e in episodes:
+        af = e.get("audio_file")
+        if not af:
+            continue
+        host = to_host_path(af)
+        if host:
+            live[host.parent.parent.resolve()] = e  # <uuid>/audio/<uuid>.mp3 -> <uuid>/
+
+    orphans: list[tuple[Path, float]] = []
+    clip_dirs: list[tuple[Path, float, str]] = []
+    skipped_young: list[str] = []
+
+    for d in sorted(EPISODES_DIR.iterdir()):
+        if not d.is_dir():
+            continue
+        d = d.resolve()
+        if d.parent != EPISODES_DIR.resolve():  # Symlink-Ausbruch o.ae.
+            continue
+        if age_minutes(d) < args.min_age:
+            skipped_young.append(d.name)
+            continue
+
+        ep = live.get(d)
+        if ep is None:
+            orphans.append((d, dir_size_mb(d)))
+            continue
+
+        clips = d / "clips"
+        final_mp3 = to_host_path(ep["audio_file"])
+        if not args.keep_clips and clips.is_dir() and final_mp3 and final_mp3.exists():
+            n = len(list(clips.glob("*")))
+            if n:
+                clip_dirs.append((clips, dir_size_mb(clips), ep.get("name") or ep["id"]))
+
+    if orphans:
+        print(f"Verwaiste Episodenordner (kein DB-Eintrag mehr) — {len(orphans)}:")
+        for d, mb in orphans:
+            print(f"  {d.name}  {mb:6.1f} MB")
+    if clip_dirs:
+        print(f"\nZwischenclips fertiger Episoden (finale MP3 existiert) — {len(clip_dirs)}:")
+        for c, mb, name in clip_dirs:
+            print(f"  {c.parent.name}/clips  {mb:6.1f} MB   ({name})")
+    if skipped_young:
+        print(f"\nUebersprungen (juenger als {args.min_age} min): {', '.join(skipped_young)}")
+
+    total = sum(mb for _, mb in orphans) + sum(mb for _, mb, _ in clip_dirs)
+    if not orphans and not clip_dirs:
+        print("Nichts aufzuraeumen.")
+        return 0
+
+    print(f"\nFreizugeben: {total:.1f} MB")
+    if not args.yes:
+        print("Trockenlauf — nichts geloescht. Mit --yes wirklich loeschen.")
+        return 0
+
+    victims = [d for d, _ in orphans] + [c for c, _, _ in clip_dirs]
+    try:
+        delete_in_container(victims)
+    except FileNotFoundError:
+        die("docker nicht gefunden — das Loeschen laeuft im Container "
+            "(die Ordner gehoeren root).")
+    except subprocess.CalledProcessError as e:
+        die(f"Loeschen im Container fehlgeschlagen: {e.stderr.decode().strip()}")
+
+    for d, _ in orphans:
+        print(f"geloescht: {d.name}")
+    for c, _, _ in clip_dirs:
+        print(f"geloescht: {c.parent.name}/clips")
+
+    rest = [p for p in victims if p.exists()]
+    if rest:
+        die(f"{len(rest)} Pfad(e) existieren noch: {', '.join(p.name for p in rest)}")
+    print(f"\n{total:.1f} MB freigegeben.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())