dschlueter/integrity_scanner_fuer_statische_Webseiten
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

perf: reuse browser-UA crawl result in cloak-check during scan

Browse source 
When cmd_scan triggers the weekly cloak-check, the browser-UA crawl
has already just been completed. Pass those pages directly to
_run_cloak_core() so only the Googlebot-UA crawl is needed,
reducing weekly scans from 3 full crawls to 2.

Standalone `cloak-check` command is unaffected (pages_normal=None).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name 2026-06-12 11:55:24 +02:00
commit 4bae633a41
1 changed files with 9 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

13
scanner/__main__.py
View file

@ -819,9 +819,14 @@ def _run_content_check(cfg: dict, snap: dict, baseline: dict) -> tuple[dict, dic
return diff, assessment, wl_violations, missing_sec
def _run_cloak_core(cfg: dict) -> tuple[dict, dict]:
"""Doppel-Crawl Browser-UA vs. Googlebot-UA. Returns (cloak_diff, assessment)."""
pages_normal, _ = _crawl_and_extract(cfg)
def _run_cloak_core(cfg: dict, pages_normal: list | None = None) -> tuple[dict, dict]:
"""Doppel-Crawl Browser-UA vs. Googlebot-UA. Returns (cloak_diff, assessment).
Wenn pages_normal übergeben wird (z.B. aus cmd_scan), wird der Browser-UA-Crawl
übersprungen und diese Seiten direkt verwendet.
"""
if pages_normal is None:
pages_normal, _ = _crawl_and_extract(cfg)
bot_cfg = {**cfg, "user_agent": GOOGLEBOT_UA}
pages_bot, _ = _crawl_and_extract(bot_cfg)
normal_map = {p["url"]: p for p in pages_normal}
@ -931,7 +936,7 @@ def cmd_scan(args: argparse.Namespace, cfg: dict) -> int:
interval = pc.get("interval_days", 7)
if pc.get("cloak_check", True) and bm.is_check_due("cloak_check", interval):
print(" Wöchentliche Tarnungs-Prüfung läuft mit ...")
cloak_diff, cloak_assess = _run_cloak_core(cfg)
cloak_diff, cloak_assess = _run_cloak_core(cfg, pages_normal=pages)
levels.append(cloak_assess["level"])
bm.mark_check_run("cloak_check")
if pc.get("assets", True) and bm.is_check_due("assets", interval):