fix: M1 exclude_paths '/' jamulix, M2 seed-should_crawl filter, M3 alert dedup

- M1: Removed '/' from jamulix.de exclude_paths (blockaded ALL paths via startswith)
- M2: Seed URLs now pass through should_crawl() in crawler.py
- M3: Alert deduplication via last_alert.json (hash of reasons+level+score)
- Added 11 tests for alert deduplication (277 total, all green)
This commit is contained in:
Dieter Schlüter 2026-06-14 21:12:36 +02:00
commit 248788e282
4 changed files with 190 additions and 4 deletions

View file

@ -2,11 +2,15 @@
Alert dispatch: e-mail (smtplib) and HTTP webhook.
Alerts are only sent when the assessment level meets or exceeds min_level.
Duplicate alerts (identical reasons since last send) are suppressed.
"""
import hashlib
import json
import logging
import os
import smtplib
from email.message import EmailMessage
from pathlib import Path
import requests
@ -16,10 +20,15 @@ from .report import render_markdown
logger = logging.getLogger(__name__)
_LEVEL_ORDER = {"green": 0, "yellow": 1, "red": 2}
_ALERT_STATE_FILE = "last_alert.json"
def send_alert(report: dict, cfg: dict) -> None:
"""Send alert if the report's level is at or above min_level."""
"""Send alert if the report's level is at or above min_level.
Duplicate alerts (identical assessment reasons since last send) are suppressed.
The dedup state is persisted in data_dir/last_alert.json per site.
"""
alerting = cfg.get("alerting", {})
level = report.get("assessment", {}).get("level", "green")
min_level = alerting.get("min_level", "yellow")
@ -28,6 +37,16 @@ def send_alert(report: dict, cfg: dict) -> None:
logger.debug("Alert suppressed: level=%s < min_level=%s", level, min_level)
return
# --- Deduplizierung: nur senden wenn sich die Alarm-Gründe geändert haben ---
alert_key = _compute_alert_key(report)
data_dir = cfg.get("data_dir")
if data_dir:
last = _load_last_alert(Path(data_dir))
if last and last.get("key") == alert_key:
logger.info("Alert suppressed (duplicate, reasons unchanged since %s)",
last.get("sent_at", "unknown"))
return
subject = _make_subject(report)
body = _format_body(report)
@ -39,6 +58,10 @@ def send_alert(report: dict, cfg: dict) -> None:
if webhook_cfg.get("enabled") and webhook_cfg.get("url"):
_send_webhook(webhook_cfg["url"], report, subject)
# Zustand speichern — für nächste Dedup-Runde
if data_dir:
_save_last_alert(Path(data_dir), alert_key)
# ---------------------------------------------------------------------------
# Formatting
@ -112,3 +135,40 @@ def _send_webhook(url: str, report: dict, subject: str) -> None:
logger.info("Webhook delivered: HTTP %s", resp.status_code)
except Exception as exc:
logger.error("Webhook alert failed: %s", exc)
# ---------------------------------------------------------------------------
# Deduplication state (last_alert.json in data_dir)
# ---------------------------------------------------------------------------
def _compute_alert_key(report: dict) -> str:
"""Stable hash of the assessment reasons + level for deduplication."""
payload = {
"level": report.get("assessment", {}).get("level", "green"),
"score": report.get("assessment", {}).get("score", 0),
"reasons": sorted(report.get("assessment", {}).get("reasons", [])),
}
return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
def _load_last_alert(data_dir: Path) -> dict:
"""Load previous alert state. Returns {} on missing/corrupt file."""
path = data_dir / _ALERT_STATE_FILE
try:
return json.loads(path.read_text())
except (FileNotFoundError, json.JSONDecodeError, OSError):
return {}
def _save_last_alert(data_dir: Path, alert_key: str) -> None:
"""Persist current alert state for next dedup round."""
from datetime import datetime, timezone
path = data_dir / _ALERT_STATE_FILE
state = {
"key": alert_key,
"sent_at": datetime.now(timezone.utc).isoformat(),
}
try:
path.write_text(json.dumps(state))
except OSError as exc:
logger.debug("Alert state write failed: %s", exc)