feat: Inhaltsinventar in SQLite — KI-Inhaltsangabe je Objekt (URL + Hash)
Vereinheitlicht den bisherigen JSON-Ledger zu EINEM abfragbaren SQLite-Store pro Site (data/content_inventory.db): jedes geprüfte Objekt mit Inhalts-Hash, URL(s), Sicherheits-Verdikt UND neutraler KI-Inhaltsangabe. - baseline.py: load/save_ai_ledger jetzt SQLite-gestützt (Dict-Schnittstelle bleibt → Analyzer unverändert). Tabellen objects + object_urls. Einmalige Migration eines vorhandenen ai_ledger.json → SQLite (.migrated). UPSERT mit first_seen-Erhalt; object_urls transaktional ersetzt. Neu: query_inventory, export_inventory_csv (beide migrieren failsafe). - ai_analyzer.py: Antwort-Schema + Prompt um neutrales Feld 'description' erweitert (im selben Call, keine Extrakosten); _make_entry speichert es. - __main__.py: neues Kommando 'inventory' (Übersicht, --search, --kind, --csv). - Doku: README + Bedienungsanleitung. Audio/Video sind im Schema (kind) vorbereitet (Phase 2: Extractor + Modalitäten). bredelar.info real migriert: 245 Objekte (200 Bilder, 45 Texte). Tests: 266 grün (+6: Round-Trip mit description, Migration, first_seen-Erhalt, invalidate, Suche/CSV-Export, description landet im Ledger). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
78dc79ceab
commit
989f5a933f
7 changed files with 262 additions and 22 deletions
|
|
@ -5,9 +5,11 @@ Snapshots are timestamped crawl results. The baseline is a manually
|
|||
approved reference state. Compromised content must never flow into the
|
||||
baseline automatically — every update requires an explicit `approve` call.
|
||||
"""
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
|
@ -234,35 +236,136 @@ class BaselineManager:
|
|||
logger.info("Asset hashes saved: %d entries", len(hashes))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# AI verdict ledger (cache keyed by content fingerprint)
|
||||
# AI content inventory (SQLite store: verdict + content description + url→hash)
|
||||
# ------------------------------------------------------------------
|
||||
#
|
||||
# Unlike the baseline, the ledger is a *cache*: it records the AI verdict
|
||||
# for every fingerprint already analysed — including "clean" ones — so that
|
||||
# unchanged content is never re-sent to the API. It lives in data_dir
|
||||
# (not baseline_dir): it is operational state, not an approved reference.
|
||||
# EINE Quelle der Wahrheit pro Site: jedes geprüfte Objekt steht mit Inhalts-Hash,
|
||||
# URL(s), Sicherheits-Verdikt UND neutraler KI-Inhaltsangabe in der DB. Der Analyzer
|
||||
# arbeitet weiterhin mit dem In-Memory-Dict {"entries": {hash:…}, "url_hashes": {url:hash}};
|
||||
# nur die Persistenz liegt in SQLite (abfragbar, skaliert, CSV-Export möglich).
|
||||
|
||||
@property
|
||||
def _ai_ledger_path(self) -> Path:
|
||||
return self.data_dir / "ai_ledger.json"
|
||||
def _content_db_path(self) -> Path:
|
||||
return self.data_dir / "content_inventory.db"
|
||||
|
||||
def _connect_inventory(self) -> sqlite3.Connection:
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(self._content_db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS objects (
|
||||
hash TEXT PRIMARY KEY,
|
||||
kind TEXT, url TEXT, description TEXT,
|
||||
category TEXT, severity TEXT, confidence REAL, explanation TEXT,
|
||||
model TEXT, dismissed INTEGER DEFAULT 0, checked_at TEXT,
|
||||
first_seen TEXT, last_seen TEXT
|
||||
)""")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS object_urls (
|
||||
url TEXT PRIMARY KEY, hash TEXT NOT NULL, last_seen TEXT
|
||||
)""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_object_urls_hash ON object_urls(hash)")
|
||||
return conn
|
||||
|
||||
def load_ai_ledger(self) -> dict:
|
||||
"""Load the AI verdict ledger. Returns {'entries': {}} if none/corrupt."""
|
||||
if not self._ai_ledger_path.exists():
|
||||
return {"entries": {}}
|
||||
"""Load the AI store into the in-memory dict the analyzer uses.
|
||||
Returns {"entries": {hash: {...}}, "url_hashes": {url: hash}}."""
|
||||
self._migrate_json_ledger()
|
||||
conn = self._connect_inventory()
|
||||
try:
|
||||
data = json.loads(self._ai_ledger_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return {"entries": {}}
|
||||
data.setdefault("entries", {})
|
||||
return data
|
||||
entries: dict = {}
|
||||
for row in conn.execute("SELECT * FROM objects"):
|
||||
d = dict(row)
|
||||
h = d.pop("hash")
|
||||
d.pop("first_seen", None)
|
||||
d.pop("last_seen", None)
|
||||
d["dismissed"] = bool(d.get("dismissed"))
|
||||
entries[h] = d
|
||||
url_hashes = {r["url"]: r["hash"]
|
||||
for r in conn.execute("SELECT url, hash FROM object_urls")}
|
||||
finally:
|
||||
conn.close()
|
||||
return {"entries": entries, "url_hashes": url_hashes}
|
||||
|
||||
def save_ai_ledger(self, ledger: dict) -> None:
|
||||
"""Persist the AI verdict ledger."""
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._ai_ledger_path.write_text(
|
||||
json.dumps(ledger, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
"""Persist the in-memory dict back to SQLite. Objekte werden geupsertet
|
||||
(first_seen bleibt erhalten); object_urls wird ersetzt (damit Entfernungen wirken)."""
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
conn = self._connect_inventory()
|
||||
try:
|
||||
for h, e in ledger.get("entries", {}).items():
|
||||
conn.execute("""
|
||||
INSERT INTO objects (hash, kind, url, description, category, severity,
|
||||
confidence, explanation, model, dismissed, checked_at, first_seen, last_seen)
|
||||
VALUES (:hash,:kind,:url,:description,:category,:severity,:confidence,
|
||||
:explanation,:model,:dismissed,:checked_at,:now,:now)
|
||||
ON CONFLICT(hash) DO UPDATE SET
|
||||
kind=excluded.kind, url=excluded.url, description=excluded.description,
|
||||
category=excluded.category, severity=excluded.severity,
|
||||
confidence=excluded.confidence, explanation=excluded.explanation,
|
||||
model=excluded.model, dismissed=excluded.dismissed,
|
||||
checked_at=excluded.checked_at, last_seen=excluded.last_seen
|
||||
""", {
|
||||
"hash": h, "kind": e.get("kind"), "url": e.get("url"),
|
||||
"description": e.get("description", ""), "category": e.get("category"),
|
||||
"severity": e.get("severity"), "confidence": e.get("confidence"),
|
||||
"explanation": e.get("explanation", ""), "model": e.get("model"),
|
||||
"dismissed": 1 if e.get("dismissed") else 0,
|
||||
"checked_at": e.get("checked_at"), "now": now,
|
||||
})
|
||||
conn.execute("DELETE FROM object_urls")
|
||||
conn.executemany(
|
||||
"INSERT INTO object_urls (url, hash, last_seen) VALUES (?,?,?)",
|
||||
[(u, h, now) for u, h in ledger.get("url_hashes", {}).items()])
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _migrate_json_ledger(self) -> None:
|
||||
"""Einmalige Migration eines alten ai_ledger.json nach SQLite."""
|
||||
old = self.data_dir / "ai_ledger.json"
|
||||
if not old.exists():
|
||||
return
|
||||
try:
|
||||
data = json.loads(old.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
old.rename(old.with_suffix(".json.corrupt"))
|
||||
return
|
||||
self.save_ai_ledger({"entries": data.get("entries", {}),
|
||||
"url_hashes": data.get("url_hashes", {})})
|
||||
old.rename(old.with_suffix(".json.migrated"))
|
||||
logger.info("ai_ledger.json nach SQLite migriert (%d Objekte).",
|
||||
len(data.get("entries", {})))
|
||||
|
||||
def query_inventory(self, search: str | None = None, kind: str | None = None) -> list[dict]:
|
||||
"""Inventar abfragen (für das `inventory`-Kommando)."""
|
||||
self._migrate_json_ledger()
|
||||
conn = self._connect_inventory()
|
||||
try:
|
||||
sql, params, cond = "SELECT * FROM objects", [], []
|
||||
if kind:
|
||||
cond.append("kind = ?"); params.append(kind)
|
||||
if search:
|
||||
cond.append("(description LIKE ? OR url LIKE ? OR category LIKE ?)")
|
||||
params += [f"%{search}%"] * 3
|
||||
if cond:
|
||||
sql += " WHERE " + " AND ".join(cond)
|
||||
sql += " ORDER BY kind, url"
|
||||
return [dict(r) for r in conn.execute(sql, params)]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def export_inventory_csv(self, path: str | Path) -> int:
|
||||
"""Inventar als CSV exportieren. Returns Anzahl Zeilen."""
|
||||
cols = ["hash", "kind", "url", "description", "category", "severity", "confidence",
|
||||
"explanation", "model", "dismissed", "checked_at", "first_seen", "last_seen"]
|
||||
rows = self.query_inventory()
|
||||
with Path(path).open("w", newline="", encoding="utf-8") as f:
|
||||
w = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore")
|
||||
w.writeheader()
|
||||
for r in rows:
|
||||
w.writerow(r)
|
||||
return len(rows)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# AI model-router state (circuit-breaker telemetry + /models catalog)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue