fix: Bugfixes, toten Code entfernt, Cache-Normalisierung & Session
Echte Bugs: - extractor: JSON-LD-Crash bei Nicht-Dict-Items (isinstance-Guard) - baseline/__main__: Crawl-Fehler ins Snapshot-Manifest -> erscheinen im Report - __main__: Whitelist-/Header-/Webshell-Checks nur noch auf status==200 - crawler: Noise-Param-Regex auf Voll-Key (view=/value= nicht mehr verworfen) - differ/__main__: unerwartetes JSON-LD via eigenem Kanal, auch auf unveraenderten Seiten erkannt, kein Re-Alert auf bereits genehmigte Typen Aufgeraeumt: - checker: check_internal_paths, find_broken_pages, canonical_is_hijacked, check_jsonld_types entfernt (nicht verdrahtet) - allowed_paths.yaml-Logik und tote Scoring-Keys entfernt - tote Imports entfernt Neuer aktiver Schutz: - Webshell-/Backdoor-Dateinamen-Check verdrahtet (suspicious_filename: 60). Regex wortgrenzen-verankert gegen Fehlalarm auf PSEMailerAntispam.js Effizienz/Struktur: - crawler nutzt requests.Session (keep-alive) - Cache-Buster-Normalisierung an einer Stelle (Extraktion) -> stabile Snapshots, differ wieder reiner Set-Diff Tests: 111 gruen (neu: test_checker.py + Regressionstests) CLAUDE.md aktualisiert Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
b3ef3a90d4
commit
68243a97b9
13 changed files with 323 additions and 153 deletions
|
|
@ -6,9 +6,22 @@ inline script patterns, JSON-LD, and links buried in HTML comments.
|
|||
"""
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from urllib.parse import urlencode, urljoin, urlparse, parse_qsl, urlunparse
|
||||
|
||||
from bs4 import BeautifulSoup, Comment
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# Query-param keys that are pure cache-busters on asset URLs (CSS/JS/img).
|
||||
# They change on every request and must be stripped before storing/diffing.
|
||||
CACHE_PARAM_KEYS = {"v", "ver", "version", "cache", "cb", "_", "_t"}
|
||||
|
||||
|
||||
def strip_cache_params(url: str) -> str:
|
||||
"""Remove cache-buster query params (e.g. style.css?v=12345) from a URL."""
|
||||
p = urlparse(url)
|
||||
if not p.query:
|
||||
return url
|
||||
kept = [(k, v) for k, v in parse_qsl(p.query) if k.lower() not in CACHE_PARAM_KEYS]
|
||||
return urlunparse(p._replace(query=urlencode(kept)))
|
||||
|
||||
# Inline-script patterns that warrant flagging
|
||||
_SUSPICIOUS_INLINE = [
|
||||
|
|
@ -134,16 +147,19 @@ def _extract_links(soup: BeautifulSoup, base_url: str) -> dict:
|
|||
rel = " ".join(tag.get("rel", []))
|
||||
if "canonical" in rel:
|
||||
result["canonical"] = u
|
||||
u = strip_cache_params(u) # assets carry cache-busters → normalise away
|
||||
result["link_rel"].append({"url": u, "rel": rel, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("script", src=True):
|
||||
u = resolve(tag["src"])
|
||||
if u:
|
||||
u = strip_cache_params(u)
|
||||
result["script"].append({"url": u, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("img", src=True):
|
||||
u = resolve(tag["src"])
|
||||
if u:
|
||||
u = strip_cache_params(u)
|
||||
result["img"].append({"url": u, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("iframe", src=True):
|
||||
|
|
@ -293,6 +309,8 @@ def _extract_jsonld(soup: BeautifulSoup) -> list[dict]:
|
|||
continue
|
||||
items = data if isinstance(data, list) else [data]
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue # JSON-LD must be an object; ignore stray scalars/arrays
|
||||
t = item.get("@type", "Unknown")
|
||||
results.append({"type": t, "preview": json.dumps(item, ensure_ascii=False)[:500]})
|
||||
return results
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue