refactor: multi-site structure — one directory per target website
Each monitored website now lives in its own named subdirectory (bredelar.info/, jamulix.de/) with a dedicated config.yaml using absolute paths. This makes adding new sites trivial: mkdir newsite.de && cp bredelar.info/config.yaml newsite.de/ # adjust target + paths, then: python -m scanner --config newsite.de/config.yaml init Changes: - bredelar.info/: moved data/, reports/, logs/, config/ from project root - bredelar.info/config.yaml: new, with absolute paths - jamulix.de/: moved from ~/Python_Programs/jamulix_check/ - jamulix.de/config.yaml: updated paths to new location - .gitignore: replaced old data/*/reports/logs entries with per-site entries - config.yaml (root): removed (replaced by site-specific configs) - CLAUDE.md: updated CLI examples to --config <site>/config.yaml Usage: python -m scanner --config bredelar.info/config.yaml scan python -m scanner --config jamulix.de/config.yaml scan Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name
parent
f964e41201
commit
ab882f912e
8 changed files with 349 additions and 137 deletions
13
.gitignore
vendored
13
.gitignore
vendored
|
|
@ -22,12 +22,13 @@ env/
|
|||
htmlcov/
|
||||
.tox/
|
||||
|
||||
# Runtime data (never commit — may contain crawled content or sensitive diffs)
|
||||
data/snapshots/
|
||||
data/baseline/
|
||||
data/pending/
|
||||
reports/
|
||||
logs/
|
||||
# Site-Daten (nie committen — enthalten Crawl-Daten und Snapshots)
|
||||
bredelar.info/data/
|
||||
bredelar.info/reports/
|
||||
bredelar.info/logs/
|
||||
jamulix.de/data/
|
||||
jamulix.de/reports/
|
||||
jamulix.de/logs/
|
||||
|
||||
# Editor
|
||||
.idea/
|
||||
|
|
|
|||
124
bredelar.info/config.yaml
Normal file
124
bredelar.info/config.yaml
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
# Integrity Scanner — Konfiguration für bredelar.info
|
||||
|
||||
target: "https://bredelar.info"
|
||||
user_agent: "integrity-scanner/1.0 (+security-monitoring)"
|
||||
request_timeout: 20
|
||||
|
||||
data_dir: "/home/dschlueter/Python_Programs/integrity_scanner_fuer_statische_Webseiten/bredelar.info/data"
|
||||
reports_dir: "/home/dschlueter/Python_Programs/integrity_scanner_fuer_statische_Webseiten/bredelar.info/reports"
|
||||
logs_dir: "/home/dschlueter/Python_Programs/integrity_scanner_fuer_statische_Webseiten/bredelar.info/logs"
|
||||
config_dir: "/home/dschlueter/Python_Programs/integrity_scanner_fuer_statische_Webseiten/bredelar.info/config"
|
||||
|
||||
crawl:
|
||||
max_pages: 200
|
||||
delay_seconds: 1.0
|
||||
respect_robots_txt: false
|
||||
# Pfade, die vom Crawl und vom Vergleich komplett ausgeschlossen werden.
|
||||
# Nützlich für Bereiche mit gewollt häufig wechselnden Inhalten (RSS-Feeds, News-Archive).
|
||||
# Beispiel: ["/rss/", "/feed/", "/news/archiv/"]
|
||||
exclude_paths: []
|
||||
|
||||
skip_extensions:
|
||||
- ".jpg"
|
||||
- ".jpeg"
|
||||
- ".png"
|
||||
- ".gif"
|
||||
- ".webp"
|
||||
- ".svg"
|
||||
- ".ico"
|
||||
- ".pdf"
|
||||
- ".zip"
|
||||
- ".woff"
|
||||
- ".woff2"
|
||||
- ".ttf"
|
||||
- ".otf"
|
||||
- ".eot"
|
||||
- ".mp4"
|
||||
- ".mp3"
|
||||
- ".webm"
|
||||
- ".avi"
|
||||
- ".mov"
|
||||
|
||||
scoring:
|
||||
new_external_domain: 50
|
||||
new_internal_url: 30
|
||||
missing_internal_url: 20
|
||||
hidden_content: 40
|
||||
unexpected_canonical: 35
|
||||
meta_refresh: 40
|
||||
unexpected_jsonld: 35
|
||||
large_text_addition: 20
|
||||
new_inline_script_suspicious: 30
|
||||
comment_links: 25
|
||||
new_broken_link: 20
|
||||
suspicious_filename: 60
|
||||
cloaking_extra_link: 60
|
||||
cloaking_extra_text: 40
|
||||
cloaking_vary_ua: 25
|
||||
changed_script: 40
|
||||
changed_stylesheet: 30
|
||||
changed_asset: 20
|
||||
|
||||
thresholds:
|
||||
yellow: 20
|
||||
red: 60
|
||||
large_text_block_chars: 200
|
||||
|
||||
# Wöchentliche Zusatzprüfungen laufen automatisch beim täglichen `scan` mit.
|
||||
periodic_checks:
|
||||
enabled: true
|
||||
interval_days: 7
|
||||
cloak_check: true
|
||||
ext_links: true
|
||||
assets: true
|
||||
|
||||
alerting:
|
||||
min_level: "yellow"
|
||||
email:
|
||||
enabled: false
|
||||
smtp_host: "localhost"
|
||||
smtp_port: 587
|
||||
smtp_tls: true
|
||||
smtp_user: ""
|
||||
smtp_password_env: "SCANNER_SMTP_PASSWORD"
|
||||
from: "scanner@bredelar.info"
|
||||
to:
|
||||
- "admin@bredelar.info"
|
||||
webhook:
|
||||
enabled: false
|
||||
url: ""
|
||||
|
||||
normalization:
|
||||
strip_html_comments: true
|
||||
collapse_whitespace: true
|
||||
ignore_selectors:
|
||||
# - "#last-modified"
|
||||
# - ".timestamp"
|
||||
ignore_patterns:
|
||||
# - '\d{1,2}\.\d{1,2}\.\d{4}'
|
||||
# - 'Stand:\s+\S+'
|
||||
|
||||
allowed_jsonld_types:
|
||||
- "Organization"
|
||||
- "WebSite"
|
||||
- "WebPage"
|
||||
- "Article"
|
||||
- "BreadcrumbList"
|
||||
- "ItemList"
|
||||
- "LocalBusiness"
|
||||
- "Place"
|
||||
- "Person"
|
||||
- "Event"
|
||||
- "FAQPage"
|
||||
- "Question"
|
||||
- "Answer"
|
||||
- "ImageObject"
|
||||
- "SiteLinksSearchBox"
|
||||
- "ContactPage"
|
||||
- "AboutPage"
|
||||
|
||||
security_headers:
|
||||
- "Content-Security-Policy"
|
||||
- "Strict-Transport-Security"
|
||||
- "X-Content-Type-Options"
|
||||
- "Referrer-Policy"
|
||||
52
bredelar.info/config/allowed_external.yaml
Normal file
52
bredelar.info/config/allowed_external.yaml
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
# Whitelist erlaubter externer Domains — bredelar.info
|
||||
# Erstellt nach initialem Crawl und manueller Sichtprüfung am 2026-06-12.
|
||||
# Neue externe Domains, die NICHT hier stehen, lösen einen Alarm aus.
|
||||
|
||||
# Lokale Vereine, Institutionen, Betriebe
|
||||
- www.bergbauspuren-bredelar.de
|
||||
- www.bergbauspuren.de
|
||||
- www.kloster-bredelar.de
|
||||
- www.schuetzen-bredelar.de
|
||||
- www.klosterschuetzen-bredelar.de
|
||||
- www.bsg-bredelar.de
|
||||
- www.sg-hoppecketal.de
|
||||
- www.sg-hoppecketal-padberg.de
|
||||
- www.feuerwehrhütte-bredelar.de
|
||||
- www.lesdurs-mc.de
|
||||
- www.kantorei-marsberg.de
|
||||
- www.tourismus-marsberg.de
|
||||
- www.katholische-kirche-marsberg.de
|
||||
- www.marsberg.ekvw.de
|
||||
|
||||
# Lokale Unternehmen (Gewerbetreibende aus bredelar.info/companies/)
|
||||
- www.VolksbankMarsberg.de
|
||||
- www.borghoff-bredelar.de
|
||||
- www.dachdeckermarsberg.de
|
||||
- www.maler-luce.de
|
||||
- www.pyls-pflasterbau.de
|
||||
|
||||
# Technische Ressourcen (Fonts, CDN, Wetter-Widget)
|
||||
- fonts.googleapis.com
|
||||
- fonts.gstatic.com
|
||||
- cdnjs.cloudflare.com
|
||||
- www.wetter.com
|
||||
- static1.wetter.com
|
||||
|
||||
# Datenschutzerklärung / Impressum (gesetzlich vorgeschriebene Links)
|
||||
- ec.europa.eu
|
||||
- www.e-recht24.de
|
||||
- www.heise.de
|
||||
- www.ratgeberrecht.eu
|
||||
- www.privacyshield.gov
|
||||
- policies.google.com
|
||||
- www.google.com
|
||||
- adssettings.google.com
|
||||
- developers.facebook.com
|
||||
- www.facebook.com
|
||||
- twitter.com
|
||||
|
||||
# Webentwickler / Technischer Betreiber
|
||||
- www.antillu.de
|
||||
|
||||
# www-Variante der eigenen Domain (wird intern verlinkt)
|
||||
- www.bredelar.info
|
||||
17
bredelar.info/config/allowed_paths.yaml
Normal file
17
bredelar.info/config/allowed_paths.yaml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# Whitelist erlaubter interner URL-Pfade (optional)
|
||||
#
|
||||
# Wenn diese Liste befüllt ist, lösen interne Links auf nicht gelistete
|
||||
# Pfade einen Alarm aus.
|
||||
# Leer lassen = keine Pfad-Whitelist-Prüfung (nur Baseline-Diff zählt).
|
||||
#
|
||||
# Format: ein Pfad pro Zeile (absoluter Pfad, ohne Domain)
|
||||
# Beispiel für bredelar.info:
|
||||
#
|
||||
# - /
|
||||
# - /index.html
|
||||
# - /kontakt/
|
||||
# - /kontakt/index.html
|
||||
# - /impressum/
|
||||
# - /datenschutz/
|
||||
# - /ueber-uns/
|
||||
# - /aktuelles/
|
||||
23
bredelar.info/config/ignore_rules.yaml
Normal file
23
bredelar.info/config/ignore_rules.yaml
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
# Normalisierungsregeln für den Diff
|
||||
#
|
||||
# Diese Einstellungen reduzieren Fehlalarme durch harmlose, automatisch
|
||||
# generierte Inhaltsänderungen.
|
||||
|
||||
# CSS-Selektoren, die vor dem Diff aus dem DOM entfernt werden.
|
||||
# Nützlich für Elemente mit automatischen Zeitstempeln oder Zählern.
|
||||
ignore_selectors:
|
||||
# - "#last-modified"
|
||||
# - ".timestamp"
|
||||
# - "[data-cache-buster]"
|
||||
# - ".cookie-banner"
|
||||
|
||||
# Reguläre Ausdrücke, die im normalisierten Text durch [IGNORED] ersetzt werden.
|
||||
# Nützlich für Datumsangaben, Versionsnummern, etc.
|
||||
ignore_patterns:
|
||||
# Datumsformate (deutsch: 12.06.2026 oder 12. Juni 2026)
|
||||
# - '\b\d{1,2}\.\s*(?:Januar|Februar|März|April|Mai|Juni|Juli|August|September|Oktober|November|Dezember)\s+\d{4}\b'
|
||||
# - '\b\d{1,2}\.\d{1,2}\.\d{4}\b'
|
||||
# "Stand: ..." Angaben
|
||||
# - 'Stand:\s+\S+'
|
||||
# Cache-Buster in URLs (werden im HTML-Text sichtbar)
|
||||
# - '\?v=[\w.]+\b'
|
||||
131
config.yaml
131
config.yaml
|
|
@ -1,131 +0,0 @@
|
|||
# Integrity Scanner — Hauptkonfiguration
|
||||
# Alle Werte sind optional; fehlende werden aus den Defaults übernommen.
|
||||
|
||||
target: "https://bredelar.info"
|
||||
user_agent: "integrity-scanner/1.0 (+security-monitoring)"
|
||||
request_timeout: 20
|
||||
|
||||
data_dir: "data"
|
||||
reports_dir: "reports"
|
||||
logs_dir: "logs"
|
||||
config_dir: "config"
|
||||
|
||||
crawl:
|
||||
max_pages: 200
|
||||
delay_seconds: 1.0
|
||||
respect_robots_txt: false
|
||||
# Pfade, die vom Crawl und vom Vergleich komplett ausgeschlossen werden.
|
||||
# Nützlich für Bereiche mit gewollt häufig wechselnden Inhalten (RSS-Feeds, News-Archive).
|
||||
# Beispiel: ["/rss/", "/feed/", "/news/archiv/"]
|
||||
# Alle URLs, deren Pfad mit einem dieser Einträge beginnt, werden ignoriert.
|
||||
exclude_paths: []
|
||||
|
||||
skip_extensions:
|
||||
- ".jpg"
|
||||
- ".jpeg"
|
||||
- ".png"
|
||||
- ".gif"
|
||||
- ".webp"
|
||||
- ".svg"
|
||||
- ".ico"
|
||||
- ".pdf"
|
||||
- ".zip"
|
||||
- ".woff"
|
||||
- ".woff2"
|
||||
- ".ttf"
|
||||
- ".otf"
|
||||
- ".eot"
|
||||
- ".mp4"
|
||||
- ".mp3"
|
||||
- ".webm"
|
||||
- ".avi"
|
||||
- ".mov"
|
||||
|
||||
scoring:
|
||||
new_external_domain: 50
|
||||
new_internal_url: 30
|
||||
missing_internal_url: 20
|
||||
hidden_content: 40
|
||||
unexpected_canonical: 35
|
||||
meta_refresh: 40
|
||||
unexpected_jsonld: 35
|
||||
large_text_addition: 20
|
||||
new_inline_script_suspicious: 30
|
||||
comment_links: 25
|
||||
new_broken_link: 20 # neue 4xx/5xx-Seite, die in Baseline nicht vorhanden war
|
||||
suspicious_filename: 60 # Link auf Webshell-/Backdoor-typischen Dateinamen
|
||||
cloaking_extra_link: 60 # Link nur im Bot-Crawl sichtbar
|
||||
cloaking_extra_text: 40 # signifikant mehr Text im Bot-Crawl (>100 Zeichen)
|
||||
cloaking_vary_ua: 25 # Vary: User-Agent Header (ohne sonstigen Unterschied)
|
||||
changed_script: 40 # JavaScript-Datei inhaltlich geändert
|
||||
changed_stylesheet: 30 # CSS-Datei inhaltlich geändert
|
||||
changed_asset: 20 # Bild / PDF / sonstige Binärdatei geändert
|
||||
|
||||
thresholds:
|
||||
yellow: 20 # ab diesem Score: Warnung (gelb)
|
||||
red: 60 # ab diesem Score: Alarm (rot)
|
||||
large_text_block_chars: 200 # Textblock ab dieser Länge wird als „groß" gewertet
|
||||
|
||||
# Wöchentliche Zusatzprüfungen laufen automatisch beim täglichen `scan` mit.
|
||||
# So genügt EIN Cron-Job für alles — kein separater Aufruf nötig.
|
||||
periodic_checks:
|
||||
enabled: true # Zusatzprüfungen automatisch mitlaufen lassen
|
||||
interval_days: 7 # wie oft (in Tagen) die Zusatzprüfungen fällig werden
|
||||
cloak_check: true # Tarnungs-Prüfung (sieht Google einen anderen Inhalt?)
|
||||
ext_links: true # externe Links auf Erreichbarkeit prüfen
|
||||
assets: true # Bilder/Skripte/Stylesheets auf Veränderung prüfen
|
||||
|
||||
alerting:
|
||||
min_level: "yellow" # "yellow" oder "red"
|
||||
email:
|
||||
enabled: false
|
||||
smtp_host: "localhost"
|
||||
smtp_port: 587
|
||||
smtp_tls: true
|
||||
smtp_user: ""
|
||||
smtp_password_env: "SCANNER_SMTP_PASSWORD" # Passwort aus Umgebungsvariable
|
||||
from: "scanner@bredelar.info"
|
||||
to:
|
||||
- "admin@bredelar.info"
|
||||
webhook:
|
||||
enabled: false
|
||||
url: "" # z.B. Telegram-Bot-Webhook, Slack-Incoming-Webhook
|
||||
|
||||
normalization:
|
||||
strip_html_comments: true
|
||||
collapse_whitespace: true
|
||||
# Selektoren, die vor dem Diff ignoriert werden (z.B. Datums-/Cache-Elemente)
|
||||
ignore_selectors:
|
||||
# - "#last-modified"
|
||||
# - ".timestamp"
|
||||
# Regex-Muster im Text, die vor dem Diff entfernt werden
|
||||
ignore_patterns:
|
||||
# - '\d{1,2}\.\d{1,2}\.\d{4}' # Datumsangaben
|
||||
# - 'Stand:\s+\S+'
|
||||
|
||||
# Erlaubte @type-Werte in JSON-LD (alles andere löst Alarm aus)
|
||||
allowed_jsonld_types:
|
||||
- "Organization"
|
||||
- "WebSite"
|
||||
- "WebPage"
|
||||
- "Article"
|
||||
- "BreadcrumbList"
|
||||
- "ItemList"
|
||||
- "LocalBusiness"
|
||||
- "Place"
|
||||
- "Person"
|
||||
- "Event"
|
||||
- "FAQPage"
|
||||
- "Question"
|
||||
- "Answer"
|
||||
- "ImageObject"
|
||||
- "SiteLinksSearchBox"
|
||||
- "ContactPage"
|
||||
- "AboutPage"
|
||||
|
||||
# Security-Header, die auf jeder Seite erwartet werden
|
||||
security_headers:
|
||||
- "Content-Security-Policy"
|
||||
- "Strict-Transport-Security"
|
||||
- "X-Content-Type-Options"
|
||||
- "Referrer-Policy"
|
||||
119
jamulix.de/config.yaml
Normal file
119
jamulix.de/config.yaml
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
# Integrity Scanner — Konfiguration für jamulix.de
|
||||
|
||||
target: "https://jamulix.de"
|
||||
user_agent: "integrity-scanner/1.0 (+security-monitoring)"
|
||||
request_timeout: 20
|
||||
|
||||
data_dir: "/home/dschlueter/Python_Programs/integrity_scanner_fuer_statische_Webseiten/jamulix.de/data"
|
||||
reports_dir: "/home/dschlueter/Python_Programs/integrity_scanner_fuer_statische_Webseiten/jamulix.de/reports"
|
||||
logs_dir: "/home/dschlueter/Python_Programs/integrity_scanner_fuer_statische_Webseiten/jamulix.de/logs"
|
||||
config_dir: "/home/dschlueter/Python_Programs/integrity_scanner_fuer_statische_Webseiten/jamulix.de/config"
|
||||
|
||||
crawl:
|
||||
max_pages: 500
|
||||
delay_seconds: 1.0
|
||||
respect_robots_txt: false
|
||||
|
||||
# /hackernews/ wird ausgeschlossen — Inhalte dort wechseln laufend (gewollt).
|
||||
exclude_paths:
|
||||
- "/hackernews/"
|
||||
|
||||
skip_extensions:
|
||||
- ".jpg"
|
||||
- ".jpeg"
|
||||
- ".png"
|
||||
- ".gif"
|
||||
- ".webp"
|
||||
- ".svg"
|
||||
- ".ico"
|
||||
- ".pdf"
|
||||
- ".zip"
|
||||
- ".woff"
|
||||
- ".woff2"
|
||||
- ".ttf"
|
||||
- ".otf"
|
||||
- ".eot"
|
||||
- ".mp4"
|
||||
- ".mp3"
|
||||
- ".webm"
|
||||
- ".avi"
|
||||
- ".mov"
|
||||
|
||||
scoring:
|
||||
new_external_domain: 50
|
||||
new_internal_url: 30
|
||||
missing_internal_url: 20
|
||||
hidden_content: 40
|
||||
unexpected_canonical: 35
|
||||
meta_refresh: 40
|
||||
unexpected_jsonld: 35
|
||||
large_text_addition: 20
|
||||
new_inline_script_suspicious: 30
|
||||
comment_links: 25
|
||||
new_broken_link: 20
|
||||
suspicious_filename: 60
|
||||
cloaking_extra_link: 60
|
||||
cloaking_extra_text: 40
|
||||
cloaking_vary_ua: 25
|
||||
changed_script: 40
|
||||
changed_stylesheet: 30
|
||||
changed_asset: 20
|
||||
|
||||
thresholds:
|
||||
yellow: 20
|
||||
red: 60
|
||||
large_text_block_chars: 200
|
||||
|
||||
periodic_checks:
|
||||
enabled: true
|
||||
interval_days: 7
|
||||
cloak_check: true
|
||||
ext_links: true
|
||||
assets: true
|
||||
|
||||
alerting:
|
||||
min_level: "yellow"
|
||||
email:
|
||||
enabled: false
|
||||
smtp_host: "localhost"
|
||||
smtp_port: 587
|
||||
smtp_tls: true
|
||||
smtp_user: ""
|
||||
smtp_password_env: "SCANNER_SMTP_PASSWORD"
|
||||
from: "scanner@jamulix.de"
|
||||
to:
|
||||
- "admin@jamulix.de"
|
||||
webhook:
|
||||
enabled: false
|
||||
url: ""
|
||||
|
||||
normalization:
|
||||
strip_html_comments: true
|
||||
collapse_whitespace: true
|
||||
ignore_selectors:
|
||||
ignore_patterns:
|
||||
|
||||
allowed_jsonld_types:
|
||||
- "Organization"
|
||||
- "WebSite"
|
||||
- "WebPage"
|
||||
- "Article"
|
||||
- "BreadcrumbList"
|
||||
- "ItemList"
|
||||
- "LocalBusiness"
|
||||
- "Place"
|
||||
- "Person"
|
||||
- "Event"
|
||||
- "FAQPage"
|
||||
- "Question"
|
||||
- "Answer"
|
||||
- "ImageObject"
|
||||
- "SiteLinksSearchBox"
|
||||
- "ContactPage"
|
||||
- "AboutPage"
|
||||
|
||||
security_headers:
|
||||
- "Content-Security-Policy"
|
||||
- "Strict-Transport-Security"
|
||||
- "X-Content-Type-Options"
|
||||
- "Referrer-Policy"
|
||||
7
jamulix.de/config/allowed_external.yaml
Normal file
7
jamulix.de/config/allowed_external.yaml
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# Erlaubte externe Domains für jamulix.de
|
||||
# Erstellt beim ersten Init-Crawl am 2026-06-12
|
||||
- kitux.de
|
||||
- cdn.tailwindcss.com
|
||||
- unpkg.com
|
||||
- fonts.googleapis.com
|
||||
- fonts.gstatic.com
|
||||
Loading…
Add table
Add a link
Reference in a new issue