feat: Inhaltsinventar in SQLite — KI-Inhaltsangabe je Objekt (URL + Hash)
Vereinheitlicht den bisherigen JSON-Ledger zu EINEM abfragbaren SQLite-Store pro Site (data/content_inventory.db): jedes geprüfte Objekt mit Inhalts-Hash, URL(s), Sicherheits-Verdikt UND neutraler KI-Inhaltsangabe. - baseline.py: load/save_ai_ledger jetzt SQLite-gestützt (Dict-Schnittstelle bleibt → Analyzer unverändert). Tabellen objects + object_urls. Einmalige Migration eines vorhandenen ai_ledger.json → SQLite (.migrated). UPSERT mit first_seen-Erhalt; object_urls transaktional ersetzt. Neu: query_inventory, export_inventory_csv (beide migrieren failsafe). - ai_analyzer.py: Antwort-Schema + Prompt um neutrales Feld 'description' erweitert (im selben Call, keine Extrakosten); _make_entry speichert es. - __main__.py: neues Kommando 'inventory' (Übersicht, --search, --kind, --csv). - Doku: README + Bedienungsanleitung. Audio/Video sind im Schema (kind) vorbereitet (Phase 2: Extractor + Modalitäten). bredelar.info real migriert: 245 Objekte (200 Bilder, 45 Texte). Tests: 266 grün (+6: Round-Trip mit description, Migration, first_seen-Erhalt, invalidate, Suche/CSV-Export, description landet im Ledger). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
78dc79ceab
commit
989f5a933f
7 changed files with 262 additions and 22 deletions
|
|
@ -991,6 +991,32 @@ def _print_diff_only(report: dict) -> None:
|
|||
print("Keine Änderungen.")
|
||||
|
||||
|
||||
def cmd_inventory(args: argparse.Namespace, cfg: dict) -> int:
|
||||
"""KI-Inhaltsinventar (Objekt → URL, Hash, Inhaltsangabe) anzeigen/suchen/exportieren."""
|
||||
bm = BaselineManager(cfg["data_dir"])
|
||||
if getattr(args, "csv", None):
|
||||
n = bm.export_inventory_csv(args.csv)
|
||||
print(f"{n} Objekt(e) nach {args.csv} exportiert.")
|
||||
return 0
|
||||
rows = bm.query_inventory(search=getattr(args, "search", None), kind=getattr(args, "kind", None))
|
||||
if not rows:
|
||||
print("Inventar ist leer (noch keine KI-Analyse gelaufen oder kein Treffer).")
|
||||
return 0
|
||||
from collections import Counter
|
||||
kinds = Counter(r["kind"] for r in rows)
|
||||
print(f"Inventar: {len(rows)} Objekt(e) — " + ", ".join(f"{k}: {v}" for k, v in sorted(kinds.items())))
|
||||
print()
|
||||
for r in rows:
|
||||
desc = (r.get("description") or "").replace("\n", " ").strip()
|
||||
if len(desc) > 90:
|
||||
desc = desc[:87] + "…"
|
||||
flag = "" if (r.get("category") or "clean") == "clean" else f" [{r['category']}]"
|
||||
print(f" [{r['kind']:5s}] {(r.get('hash') or '')[:12]} {r.get('url', '')}{flag}")
|
||||
if desc:
|
||||
print(f" {desc}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_report(args: argparse.Namespace, cfg: dict) -> int:
|
||||
rd = Path(cfg["reports_dir"])
|
||||
if not rd.exists():
|
||||
|
|
@ -1352,6 +1378,12 @@ def _build_parser() -> argparse.ArgumentParser:
|
|||
ad.add_argument("--hash", help="Nur diesen Fingerprint quittieren")
|
||||
ad.add_argument("--all", action="store_true", help="Alle offenen KI-Funde quittieren")
|
||||
|
||||
iv = sub.add_parser("inventory", help="KI-Inhaltsinventar anzeigen/durchsuchen/exportieren")
|
||||
iv.add_argument("--search", help="Volltextsuche über Inhaltsangabe/URL/Kategorie")
|
||||
iv.add_argument("--kind", choices=["text", "image", "audio", "video"],
|
||||
help="Nur diesen Objekttyp anzeigen")
|
||||
iv.add_argument("--csv", help="Inventar als CSV-Datei exportieren (Pfad)")
|
||||
|
||||
return p
|
||||
|
||||
|
||||
|
|
@ -1433,6 +1465,7 @@ def main() -> None:
|
|||
"scan": cmd_scan,
|
||||
"approve": cmd_approve,
|
||||
"report": cmd_report,
|
||||
"inventory": cmd_inventory,
|
||||
"status": cmd_status,
|
||||
"cloak-check": cmd_cloak_check,
|
||||
"check-ext-links": cmd_check_ext_links,
|
||||
|
|
|
|||
|
|
@ -58,12 +58,13 @@ _RESPONSE_SCHEMA = {
|
|||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"description": {"type": "string"},
|
||||
"category": {"type": "string", "enum": _CATEGORIES},
|
||||
"severity": {"type": "string", "enum": ["none", "low", "medium", "high"]},
|
||||
"confidence": {"type": "number"},
|
||||
"explanation": {"type": "string"},
|
||||
},
|
||||
"required": ["category", "severity", "confidence", "explanation"],
|
||||
"required": ["description", "category", "severity", "confidence", "explanation"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
|
|
@ -483,6 +484,7 @@ def _make_entry(kind: str, url: str, verdict: dict, model: str) -> dict:
|
|||
return {
|
||||
"kind": kind,
|
||||
"url": url,
|
||||
"description": verdict.get("description", ""),
|
||||
"category": verdict.get("category", "clean"),
|
||||
"severity": verdict.get("severity", "none"),
|
||||
"confidence": float(verdict.get("confidence", 0.0)),
|
||||
|
|
@ -566,7 +568,9 @@ _SYSTEM_PROMPT = (
|
|||
"einen Werbe- oder Spam-Text nicht unproblematisch — ein Angreifer kann sie "
|
||||
"gezielt einstreuen. Bewerte den Gesamtcharakter des Inhalts.\n"
|
||||
"Gib 'clean' zurück, wenn der Inhalt zum Thema passt und unproblematisch ist. "
|
||||
"Antworte ausschließlich im vorgegebenen JSON-Format mit deutscher Begründung."
|
||||
"Gib im Feld 'description' eine knappe, neutrale Inhaltsangabe (1–2 Sätze, deutsch): "
|
||||
"was ist auf dem Bild zu sehen bzw. worum geht es im Text — sachlich, unabhängig vom "
|
||||
"Sicherheitsurteil. Antworte ausschließlich im vorgegebenen JSON-Format."
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,9 +5,11 @@ Snapshots are timestamped crawl results. The baseline is a manually
|
|||
approved reference state. Compromised content must never flow into the
|
||||
baseline automatically — every update requires an explicit `approve` call.
|
||||
"""
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
|
@ -234,35 +236,136 @@ class BaselineManager:
|
|||
logger.info("Asset hashes saved: %d entries", len(hashes))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# AI verdict ledger (cache keyed by content fingerprint)
|
||||
# AI content inventory (SQLite store: verdict + content description + url→hash)
|
||||
# ------------------------------------------------------------------
|
||||
#
|
||||
# Unlike the baseline, the ledger is a *cache*: it records the AI verdict
|
||||
# for every fingerprint already analysed — including "clean" ones — so that
|
||||
# unchanged content is never re-sent to the API. It lives in data_dir
|
||||
# (not baseline_dir): it is operational state, not an approved reference.
|
||||
# EINE Quelle der Wahrheit pro Site: jedes geprüfte Objekt steht mit Inhalts-Hash,
|
||||
# URL(s), Sicherheits-Verdikt UND neutraler KI-Inhaltsangabe in der DB. Der Analyzer
|
||||
# arbeitet weiterhin mit dem In-Memory-Dict {"entries": {hash:…}, "url_hashes": {url:hash}};
|
||||
# nur die Persistenz liegt in SQLite (abfragbar, skaliert, CSV-Export möglich).
|
||||
|
||||
@property
|
||||
def _ai_ledger_path(self) -> Path:
|
||||
return self.data_dir / "ai_ledger.json"
|
||||
def _content_db_path(self) -> Path:
|
||||
return self.data_dir / "content_inventory.db"
|
||||
|
||||
def _connect_inventory(self) -> sqlite3.Connection:
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(self._content_db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS objects (
|
||||
hash TEXT PRIMARY KEY,
|
||||
kind TEXT, url TEXT, description TEXT,
|
||||
category TEXT, severity TEXT, confidence REAL, explanation TEXT,
|
||||
model TEXT, dismissed INTEGER DEFAULT 0, checked_at TEXT,
|
||||
first_seen TEXT, last_seen TEXT
|
||||
)""")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS object_urls (
|
||||
url TEXT PRIMARY KEY, hash TEXT NOT NULL, last_seen TEXT
|
||||
)""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_object_urls_hash ON object_urls(hash)")
|
||||
return conn
|
||||
|
||||
def load_ai_ledger(self) -> dict:
|
||||
"""Load the AI verdict ledger. Returns {'entries': {}} if none/corrupt."""
|
||||
if not self._ai_ledger_path.exists():
|
||||
return {"entries": {}}
|
||||
"""Load the AI store into the in-memory dict the analyzer uses.
|
||||
Returns {"entries": {hash: {...}}, "url_hashes": {url: hash}}."""
|
||||
self._migrate_json_ledger()
|
||||
conn = self._connect_inventory()
|
||||
try:
|
||||
data = json.loads(self._ai_ledger_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return {"entries": {}}
|
||||
data.setdefault("entries", {})
|
||||
return data
|
||||
entries: dict = {}
|
||||
for row in conn.execute("SELECT * FROM objects"):
|
||||
d = dict(row)
|
||||
h = d.pop("hash")
|
||||
d.pop("first_seen", None)
|
||||
d.pop("last_seen", None)
|
||||
d["dismissed"] = bool(d.get("dismissed"))
|
||||
entries[h] = d
|
||||
url_hashes = {r["url"]: r["hash"]
|
||||
for r in conn.execute("SELECT url, hash FROM object_urls")}
|
||||
finally:
|
||||
conn.close()
|
||||
return {"entries": entries, "url_hashes": url_hashes}
|
||||
|
||||
def save_ai_ledger(self, ledger: dict) -> None:
|
||||
"""Persist the AI verdict ledger."""
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._ai_ledger_path.write_text(
|
||||
json.dumps(ledger, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
"""Persist the in-memory dict back to SQLite. Objekte werden geupsertet
|
||||
(first_seen bleibt erhalten); object_urls wird ersetzt (damit Entfernungen wirken)."""
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
conn = self._connect_inventory()
|
||||
try:
|
||||
for h, e in ledger.get("entries", {}).items():
|
||||
conn.execute("""
|
||||
INSERT INTO objects (hash, kind, url, description, category, severity,
|
||||
confidence, explanation, model, dismissed, checked_at, first_seen, last_seen)
|
||||
VALUES (:hash,:kind,:url,:description,:category,:severity,:confidence,
|
||||
:explanation,:model,:dismissed,:checked_at,:now,:now)
|
||||
ON CONFLICT(hash) DO UPDATE SET
|
||||
kind=excluded.kind, url=excluded.url, description=excluded.description,
|
||||
category=excluded.category, severity=excluded.severity,
|
||||
confidence=excluded.confidence, explanation=excluded.explanation,
|
||||
model=excluded.model, dismissed=excluded.dismissed,
|
||||
checked_at=excluded.checked_at, last_seen=excluded.last_seen
|
||||
""", {
|
||||
"hash": h, "kind": e.get("kind"), "url": e.get("url"),
|
||||
"description": e.get("description", ""), "category": e.get("category"),
|
||||
"severity": e.get("severity"), "confidence": e.get("confidence"),
|
||||
"explanation": e.get("explanation", ""), "model": e.get("model"),
|
||||
"dismissed": 1 if e.get("dismissed") else 0,
|
||||
"checked_at": e.get("checked_at"), "now": now,
|
||||
})
|
||||
conn.execute("DELETE FROM object_urls")
|
||||
conn.executemany(
|
||||
"INSERT INTO object_urls (url, hash, last_seen) VALUES (?,?,?)",
|
||||
[(u, h, now) for u, h in ledger.get("url_hashes", {}).items()])
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _migrate_json_ledger(self) -> None:
|
||||
"""Einmalige Migration eines alten ai_ledger.json nach SQLite."""
|
||||
old = self.data_dir / "ai_ledger.json"
|
||||
if not old.exists():
|
||||
return
|
||||
try:
|
||||
data = json.loads(old.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
old.rename(old.with_suffix(".json.corrupt"))
|
||||
return
|
||||
self.save_ai_ledger({"entries": data.get("entries", {}),
|
||||
"url_hashes": data.get("url_hashes", {})})
|
||||
old.rename(old.with_suffix(".json.migrated"))
|
||||
logger.info("ai_ledger.json nach SQLite migriert (%d Objekte).",
|
||||
len(data.get("entries", {})))
|
||||
|
||||
def query_inventory(self, search: str | None = None, kind: str | None = None) -> list[dict]:
|
||||
"""Inventar abfragen (für das `inventory`-Kommando)."""
|
||||
self._migrate_json_ledger()
|
||||
conn = self._connect_inventory()
|
||||
try:
|
||||
sql, params, cond = "SELECT * FROM objects", [], []
|
||||
if kind:
|
||||
cond.append("kind = ?"); params.append(kind)
|
||||
if search:
|
||||
cond.append("(description LIKE ? OR url LIKE ? OR category LIKE ?)")
|
||||
params += [f"%{search}%"] * 3
|
||||
if cond:
|
||||
sql += " WHERE " + " AND ".join(cond)
|
||||
sql += " ORDER BY kind, url"
|
||||
return [dict(r) for r in conn.execute(sql, params)]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def export_inventory_csv(self, path: str | Path) -> int:
|
||||
"""Inventar als CSV exportieren. Returns Anzahl Zeilen."""
|
||||
cols = ["hash", "kind", "url", "description", "category", "severity", "confidence",
|
||||
"explanation", "model", "dismissed", "checked_at", "first_seen", "last_seen"]
|
||||
rows = self.query_inventory()
|
||||
with Path(path).open("w", newline="", encoding="utf-8") as f:
|
||||
w = csv.DictWriter(f, fieldnames=cols, extrasaction="ignore")
|
||||
w.writeheader()
|
||||
for r in rows:
|
||||
w.writerow(r)
|
||||
return len(rows)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# AI model-router state (circuit-breaker telemetry + /models catalog)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue