fix: Bugfixes, toten Code entfernt, Cache-Normalisierung & Session
Echte Bugs: - extractor: JSON-LD-Crash bei Nicht-Dict-Items (isinstance-Guard) - baseline/__main__: Crawl-Fehler ins Snapshot-Manifest -> erscheinen im Report - __main__: Whitelist-/Header-/Webshell-Checks nur noch auf status==200 - crawler: Noise-Param-Regex auf Voll-Key (view=/value= nicht mehr verworfen) - differ/__main__: unerwartetes JSON-LD via eigenem Kanal, auch auf unveraenderten Seiten erkannt, kein Re-Alert auf bereits genehmigte Typen Aufgeraeumt: - checker: check_internal_paths, find_broken_pages, canonical_is_hijacked, check_jsonld_types entfernt (nicht verdrahtet) - allowed_paths.yaml-Logik und tote Scoring-Keys entfernt - tote Imports entfernt Neuer aktiver Schutz: - Webshell-/Backdoor-Dateinamen-Check verdrahtet (suspicious_filename: 60). Regex wortgrenzen-verankert gegen Fehlalarm auf PSEMailerAntispam.js Effizienz/Struktur: - crawler nutzt requests.Session (keep-alive) - Cache-Buster-Normalisierung an einer Stelle (Extraktion) -> stabile Snapshots, differ wieder reiner Set-Diff Tests: 111 gruen (neu: test_checker.py + Regressionstests) CLAUDE.md aktualisiert Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Your Name
parent
b3ef3a90d4
commit
68243a97b9
13 changed files with 323 additions and 153 deletions
|
|
@ -24,8 +24,6 @@ from urllib.parse import urlparse
|
|||
from .alerter import send_alert
|
||||
from .baseline import BaselineManager
|
||||
from .checker import (
|
||||
canonical_is_hijacked,
|
||||
check_jsonld_types,
|
||||
check_links_against_whitelist,
|
||||
check_security_headers,
|
||||
is_suspicious_url,
|
||||
|
|
@ -33,7 +31,7 @@ from .checker import (
|
|||
)
|
||||
from .config import load_config, resolve_paths
|
||||
from .crawler import Crawler
|
||||
from .differ import compare_snapshots, normalize_text, score_diff
|
||||
from .differ import compare_snapshots, score_diff
|
||||
from .extractor import extract_page
|
||||
|
||||
|
||||
|
|
@ -82,11 +80,9 @@ def _crawl_and_extract(cfg: dict) -> tuple[list[dict], list[dict]]:
|
|||
# Whitelist loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_whitelists(cfg: dict) -> tuple[set[str], set[str]]:
|
||||
def _load_allowed_external(cfg: dict) -> set[str]:
|
||||
config_dir = Path(cfg["config_dir"])
|
||||
allowed_ext = load_whitelist(config_dir / "allowed_external.yaml")
|
||||
allowed_int = load_whitelist(config_dir / "allowed_paths.yaml")
|
||||
return allowed_ext, allowed_int
|
||||
return load_whitelist(config_dir / "allowed_external.yaml")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -154,6 +150,8 @@ def _render_markdown(r: dict) -> str:
|
|||
f"| Neue externe Domains | {len(d.get('new_external_domains', []))} |",
|
||||
f"| Kaputte Links (neu) | {len(d.get('new_broken_urls', []))} |",
|
||||
f"| Kaputte Links (bekannt) | {len(d.get('known_broken_urls', []))} |",
|
||||
f"| Unerwartetes JSON-LD | {len(d.get('unexpected_jsonld', []))} |",
|
||||
f"| Verdächtige Dateinamen | {len(d.get('suspicious_filenames', []))} |",
|
||||
f"| Whitelist-Verstösse | {len(r.get('whitelist_violations', []))} |",
|
||||
f"| Crawl-Fehler | {len(r.get('crawl_errors', []))} |",
|
||||
"",
|
||||
|
|
@ -244,6 +242,16 @@ def _render_markdown(r: dict) -> str:
|
|||
for entry in known_broken:
|
||||
lines.append(f"- [bekannt] HTTP {entry['status']} `{entry['url']}`")
|
||||
|
||||
if d.get("unexpected_jsonld"):
|
||||
lines += ["", "## Unerwartetes JSON-LD", ""]
|
||||
for entry in d["unexpected_jsonld"]:
|
||||
lines.append(f"- `{entry['url']}`: @type={entry['type']}")
|
||||
|
||||
if d.get("suspicious_filenames"):
|
||||
lines += ["", "## Verdächtige Dateinamen", ""]
|
||||
for entry in d["suspicious_filenames"]:
|
||||
lines.append(f"- `{entry['page']}` → `{entry['url']}` ({entry['link_type']})")
|
||||
|
||||
if r.get("whitelist_violations"):
|
||||
lines += ["", "## Whitelist-Verstösse", ""]
|
||||
for v in r["whitelist_violations"][:50]:
|
||||
|
|
@ -311,14 +319,13 @@ def cmd_init(args: argparse.Namespace, cfg: dict) -> int:
|
|||
return 1
|
||||
|
||||
bm = BaselineManager(cfg["data_dir"])
|
||||
snap_dir = bm.save_snapshot(pages)
|
||||
snap_dir = bm.save_snapshot(pages, errors)
|
||||
print(f"Snapshot gespeichert: {snap_dir}")
|
||||
print(f" Seiten: {len(pages)}")
|
||||
print(f" Fehler: {len(errors)}")
|
||||
print()
|
||||
|
||||
allowed_ext, allowed_int = _load_whitelists(cfg)
|
||||
base_netloc = urlparse(cfg["target"]).netloc
|
||||
allowed_ext = _load_allowed_external(cfg)
|
||||
|
||||
print("=== Gefundene interne URLs ===")
|
||||
for p in pages:
|
||||
|
|
@ -383,7 +390,7 @@ def cmd_crawl(args: argparse.Namespace, cfg: dict) -> int:
|
|||
print(f"Crawle {cfg['target']} ...")
|
||||
pages, errors = _crawl_and_extract(cfg)
|
||||
bm = BaselineManager(cfg["data_dir"])
|
||||
snap_dir = bm.save_snapshot(pages)
|
||||
snap_dir = bm.save_snapshot(pages, errors)
|
||||
print(f"Snapshot: {snap_dir}")
|
||||
print(f" Seiten: {len(pages)}")
|
||||
print(f" Fehler: {len(errors)}")
|
||||
|
|
@ -418,13 +425,18 @@ def cmd_check(args: argparse.Namespace, cfg: dict) -> int:
|
|||
|
||||
diff = compare_snapshots(baseline["pages"], snap["pages"], cfg)
|
||||
|
||||
# Whitelist checks on all current pages
|
||||
allowed_ext, allowed_int = _load_whitelists(cfg)
|
||||
# Whitelist / header / webshell checks — only on successfully fetched HTML
|
||||
# pages (status 200). 404 / asset responses would otherwise create noise.
|
||||
allowed_ext = _load_allowed_external(cfg)
|
||||
base_netloc = urlparse(cfg["target"]).netloc
|
||||
wl_violations: list[dict] = []
|
||||
missing_sec: dict = {}
|
||||
suspicious_filenames: list[dict] = []
|
||||
|
||||
for page in snap["pages"].values():
|
||||
if page.get("status") != 200 or not page.get("links"):
|
||||
continue
|
||||
|
||||
violations = check_links_against_whitelist(page, allowed_ext, base_netloc)
|
||||
if violations:
|
||||
wl_violations.extend({"page": page["url"], **v} for v in violations)
|
||||
|
|
@ -434,20 +446,19 @@ def cmd_check(args: argparse.Namespace, cfg: dict) -> int:
|
|||
if missing:
|
||||
missing_sec[page["url"]] = missing
|
||||
|
||||
# Unexpected JSON-LD types (absolute check, not just diff-based)
|
||||
unexp = check_jsonld_types(page, cfg.get("allowed_jsonld_types", []))
|
||||
for j in unexp:
|
||||
if not any(
|
||||
pd.get("url") == page["url"] and j in pd.get("new_jsonld", [])
|
||||
for pd in diff.get("page_diffs", [])
|
||||
):
|
||||
# Add as new_jsonld to the page_diff if not already there
|
||||
pd_for_url = next(
|
||||
(pd for pd in diff.get("page_diffs", []) if pd["url"] == page["url"]),
|
||||
None,
|
||||
)
|
||||
if pd_for_url:
|
||||
pd_for_url.setdefault("new_jsonld", []).append({**j, "unexpected": True})
|
||||
# Webshell / backdoor filename in any referenced link
|
||||
for ltype, links in page.get("links", {}).items():
|
||||
if not isinstance(links, list):
|
||||
continue
|
||||
for l in links:
|
||||
if isinstance(l, dict) and is_suspicious_url(l.get("url", "")):
|
||||
suspicious_filenames.append({
|
||||
"page": page["url"],
|
||||
"url": l["url"],
|
||||
"link_type": ltype,
|
||||
})
|
||||
|
||||
diff["suspicious_filenames"] = suspicious_filenames
|
||||
|
||||
# Score with whitelist-violation contribution
|
||||
for v in wl_violations:
|
||||
|
|
@ -460,10 +471,7 @@ def cmd_check(args: argparse.Namespace, cfg: dict) -> int:
|
|||
|
||||
assessment = score_diff(diff, cfg)
|
||||
|
||||
crawl_errors: list[dict] = []
|
||||
if getattr(args, "with_errors", False):
|
||||
# errors were not persisted in snapshot, skip
|
||||
pass
|
||||
crawl_errors = snap.get("errors", [])
|
||||
|
||||
report = _build_report(
|
||||
cfg["target"], diff, assessment, wl_violations, missing_sec,
|
||||
|
|
@ -485,7 +493,7 @@ def cmd_scan(args: argparse.Namespace, cfg: dict) -> int:
|
|||
print(f"Scanne {cfg['target']} ...")
|
||||
pages, errors = _crawl_and_extract(cfg)
|
||||
bm = BaselineManager(cfg["data_dir"])
|
||||
snap_dir = bm.save_snapshot(pages)
|
||||
snap_dir = bm.save_snapshot(pages, errors)
|
||||
logger.info("Snapshot: %s (%d Seiten, %d Fehler)", snap_dir, len(pages), len(errors))
|
||||
|
||||
if not bm.baseline_exists():
|
||||
|
|
|
|||
|
|
@ -31,9 +31,11 @@ class BaselineManager:
|
|||
# Snapshot operations
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def save_snapshot(self, pages: list[dict]) -> Path:
|
||||
def save_snapshot(self, pages: list[dict], errors: list[dict] | None = None) -> Path:
|
||||
"""
|
||||
Persist a crawl+extraction result as a timestamped snapshot.
|
||||
Crawl errors (network failures) are recorded in the manifest so they
|
||||
survive into the comparison/report stage.
|
||||
Returns the snapshot directory path.
|
||||
"""
|
||||
ts = _utc_stamp()
|
||||
|
|
@ -45,6 +47,7 @@ class BaselineManager:
|
|||
"timestamp": ts,
|
||||
"page_count": len(pages),
|
||||
"urls": [p["url"] for p in pages],
|
||||
"errors": errors or [],
|
||||
}
|
||||
(snap_dir / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
|
|
@ -71,7 +74,7 @@ class BaselineManager:
|
|||
def load_snapshot(self, snap_dir: Optional[Path] = None) -> dict:
|
||||
"""
|
||||
Load a snapshot (default: latest).
|
||||
Returns {manifest, pages: {url: page_dict}, dir: Path}.
|
||||
Returns {manifest, pages: {url: page_dict}, errors: list, dir: Path}.
|
||||
"""
|
||||
if snap_dir is None:
|
||||
snap_dir = self.latest_snapshot_dir()
|
||||
|
|
@ -85,7 +88,12 @@ class BaselineManager:
|
|||
page = json.loads(f.read_text(encoding="utf-8"))
|
||||
pages[page["url"]] = page
|
||||
|
||||
return {"manifest": manifest, "pages": pages, "dir": snap_dir}
|
||||
return {
|
||||
"manifest": manifest,
|
||||
"pages": pages,
|
||||
"errors": manifest.get("errors", []),
|
||||
"dir": snap_dir,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Baseline operations
|
||||
|
|
|
|||
|
|
@ -11,8 +11,11 @@ from urllib.parse import urlparse
|
|||
|
||||
import yaml
|
||||
|
||||
# Webshell / backdoor filename markers. Word boundaries are essential:
|
||||
# without them "mailer" matches the legitimate "PSEMailerAntispam.js" and
|
||||
# "gate" matches "navigate", producing false positives on every scan.
|
||||
_SUSPICIOUS_FILENAME_RE = re.compile(
|
||||
r"(shell|backdoor|cmd|upload|gate|mailer|wso|c99|r57|b374k)",
|
||||
r"\b(shell|backdoor|webshell|c99|r57|wso|b374k|cmd|reverse_shell|bindshell)\b",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
|
@ -87,30 +90,6 @@ def check_links_against_whitelist(
|
|||
return violations
|
||||
|
||||
|
||||
def check_internal_paths(
|
||||
page: dict,
|
||||
allowed_paths: set[str],
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Flag internal links to paths not in the allowed_paths whitelist.
|
||||
If allowed_paths is empty, the check is skipped.
|
||||
"""
|
||||
if not allowed_paths:
|
||||
return []
|
||||
violations: list[dict] = []
|
||||
for link in page.get("links", {}).get("a", []):
|
||||
if link.get("class") != "internal":
|
||||
continue
|
||||
path = urlparse(link.get("url", "")).path
|
||||
if path and path not in allowed_paths:
|
||||
violations.append({
|
||||
"type": "unlisted_internal_path",
|
||||
"url": link.get("url", ""),
|
||||
"path": path,
|
||||
})
|
||||
return violations
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Security headers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -122,38 +101,6 @@ def check_security_headers(response_headers: dict, required: list[str]) -> list[
|
|||
return [h for h in required if h.lower() not in present]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Broken links
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def find_broken_pages(pages: dict) -> list[dict]:
|
||||
"""Return all pages whose HTTP status is >= 400 (broken / server error)."""
|
||||
return [
|
||||
{"url": p["url"], "status": p.get("status", 0)}
|
||||
for p in pages.values()
|
||||
if p.get("status", 0) >= 400
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Canonical hijack
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def canonical_is_hijacked(page: dict, base_netloc: str) -> bool:
|
||||
"""
|
||||
Return True if the canonical URL points to a different domain.
|
||||
Canonical appearing in both <link rel=canonical> and metadata is checked.
|
||||
"""
|
||||
canonical = (
|
||||
page.get("links", {}).get("canonical")
|
||||
or page.get("metadata", {}).get("canonical")
|
||||
)
|
||||
if not canonical:
|
||||
return False
|
||||
netloc = urlparse(canonical).netloc
|
||||
return bool(netloc) and netloc != base_netloc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Suspicious filenames
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -162,16 +109,3 @@ def is_suspicious_url(url: str) -> bool:
|
|||
"""Flag URLs with names typical for webshells / backdoors."""
|
||||
filename = urlparse(url).path.rsplit("/", 1)[-1]
|
||||
return bool(_SUSPICIOUS_FILENAME_RE.search(filename))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON-LD type check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_jsonld_types(page: dict, allowed_types: list[str]) -> list[dict]:
|
||||
"""Return JSON-LD entries whose @type is not in the allowed list."""
|
||||
allowed = set(allowed_types)
|
||||
return [
|
||||
j for j in page.get("jsonld", [])
|
||||
if j.get("type") not in allowed
|
||||
]
|
||||
|
|
|
|||
|
|
@ -30,12 +30,9 @@ DEFAULT_CONFIG: dict = {
|
|||
"unexpected_jsonld": 35,
|
||||
"large_text_addition": 20,
|
||||
"new_inline_script_suspicious": 30,
|
||||
"missing_security_header": 5,
|
||||
"suspicious_content_pattern": 50,
|
||||
"comment_links": 25,
|
||||
"noscript_links": 25,
|
||||
"new_external_link_unlisted": 40,
|
||||
"new_broken_link": 20,
|
||||
"suspicious_filename": 60,
|
||||
},
|
||||
"thresholds": {
|
||||
"yellow": 20,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
"""Pure-Python crawler — no wget dependency."""
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlencode, urljoin, urlparse, urlunparse
|
||||
|
||||
|
|
@ -9,11 +8,20 @@ from bs4 import BeautifulSoup
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Query parameters that indicate calendar/session/tracking noise
|
||||
_NOISE_PARAM_RE = re.compile(
|
||||
r"^(year|month|day|date|week|session|token|sid|nonce|_|v|cache|utm_|fbclid)",
|
||||
re.I,
|
||||
)
|
||||
# Query-param keys that indicate calendar / session / tracking / cache noise.
|
||||
# Matched as whole keys (not prefixes) so legitimate params like ?view= or
|
||||
# ?value= are never silently dropped. utm_* is a known tracking prefix family.
|
||||
_NOISE_PARAM_KEYS = {
|
||||
"year", "month", "day", "date", "week",
|
||||
"session", "token", "sid", "nonce", "fbclid",
|
||||
"v", "ver", "version", "cache", "cb", "_", "_t",
|
||||
}
|
||||
|
||||
|
||||
def _is_noise_param(key: str) -> bool:
|
||||
"""True if a query-param key is calendar/session/tracking/cache noise."""
|
||||
k = key.lower()
|
||||
return k in _NOISE_PARAM_KEYS or k.startswith("utm_")
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str | None:
|
||||
|
|
@ -36,7 +44,7 @@ def normalize_url(url: str) -> str | None:
|
|||
(k, v)
|
||||
for k, vs in parse_qs(p.query, keep_blank_values=True).items()
|
||||
for v in vs
|
||||
if not _NOISE_PARAM_RE.match(k)
|
||||
if not _is_noise_param(k)
|
||||
)
|
||||
p = p._replace(query=urlencode(clean_params))
|
||||
return urlunparse(p)
|
||||
|
|
@ -53,8 +61,8 @@ def should_crawl(url: str, base_netloc: str, skip_extensions: list[str]) -> bool
|
|||
for ext in skip_extensions:
|
||||
if path_lower.endswith(ext):
|
||||
return False
|
||||
# Skip URLs whose path looks like calendar/session noise
|
||||
if _NOISE_PARAM_RE.search(p.query):
|
||||
# Skip URLs whose query carries calendar/session/tracking noise
|
||||
if p.query and any(_is_noise_param(k) for k in parse_qs(p.query)):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
|
@ -68,6 +76,8 @@ class Crawler:
|
|||
self.skip_ext: list[str] = cfg["crawl"]["skip_extensions"]
|
||||
self.timeout: int = cfg["request_timeout"]
|
||||
self.headers = {"User-Agent": cfg["user_agent"]}
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(self.headers)
|
||||
|
||||
def crawl(self) -> dict:
|
||||
"""
|
||||
|
|
@ -82,6 +92,30 @@ class Crawler:
|
|||
pages: list[dict] = []
|
||||
errors: list[dict] = []
|
||||
|
||||
try:
|
||||
self._crawl_loop(queue, seen, pages, errors)
|
||||
finally:
|
||||
self.session.close()
|
||||
|
||||
logger.info(
|
||||
"Crawl finished: %d pages, %d errors, %d queued but skipped",
|
||||
len(pages),
|
||||
len(errors),
|
||||
len(queue),
|
||||
)
|
||||
return {
|
||||
"target": self.target,
|
||||
"pages": pages,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
def _crawl_loop(
|
||||
self,
|
||||
queue: list[str],
|
||||
seen: set[str],
|
||||
pages: list[dict],
|
||||
errors: list[dict],
|
||||
) -> None:
|
||||
while queue and len(pages) < self.max_pages:
|
||||
url = queue.pop(0)
|
||||
if url in seen:
|
||||
|
|
@ -93,9 +127,8 @@ class Crawler:
|
|||
|
||||
logger.debug("Crawling %s", url)
|
||||
try:
|
||||
resp = requests.get(
|
||||
resp = self.session.get(
|
||||
url,
|
||||
headers=self.headers,
|
||||
timeout=self.timeout,
|
||||
allow_redirects=True,
|
||||
)
|
||||
|
|
@ -123,18 +156,6 @@ class Crawler:
|
|||
errors.append({"url": url, "error": str(exc)})
|
||||
logger.warning("Error crawling %s: %s", url, exc)
|
||||
|
||||
logger.info(
|
||||
"Crawl finished: %d pages, %d errors, %d queued but skipped",
|
||||
len(pages),
|
||||
len(errors),
|
||||
len(queue),
|
||||
)
|
||||
return {
|
||||
"target": self.target,
|
||||
"pages": pages,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
def _extract_follow_links(self, html: str, base_url: str) -> list[str]:
|
||||
"""Extract internal <a href> links worth following."""
|
||||
result = []
|
||||
|
|
|
|||
|
|
@ -43,7 +43,11 @@ def diff_text(old: str, new: str, context: int = 2) -> list[str]:
|
|||
|
||||
|
||||
def diff_link_set(old_list: list[dict], new_list: list[dict]) -> dict:
|
||||
"""Compare two link lists by URL."""
|
||||
"""Compare two link lists by URL.
|
||||
|
||||
Cache-buster query params are stripped at extraction time
|
||||
(extractor.strip_cache_params), so stored URLs are already stable here.
|
||||
"""
|
||||
old_urls = {l["url"] for l in old_list if isinstance(l, dict) and "url" in l}
|
||||
new_urls = {l["url"] for l in new_list if isinstance(l, dict) and "url" in l}
|
||||
return {
|
||||
|
|
@ -228,6 +232,20 @@ def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> d
|
|||
if url in current_pages
|
||||
]
|
||||
|
||||
# Unexpected JSON-LD @types — independent of page_diffs.
|
||||
# Only flag types that are NOT allowed AND were not already in the baseline
|
||||
# for that URL (avoids re-alerting on a pre-approved page).
|
||||
allowed_jsonld = set(cfg.get("allowed_jsonld_types", []))
|
||||
unexpected_jsonld: list[dict] = []
|
||||
for url, page in current_pages.items():
|
||||
base_types = {
|
||||
j.get("type") for j in baseline_pages.get(url, {}).get("jsonld", [])
|
||||
}
|
||||
for j in page.get("jsonld", []):
|
||||
t = j.get("type")
|
||||
if t not in allowed_jsonld and t not in base_types:
|
||||
unexpected_jsonld.append({"url": url, "type": t})
|
||||
|
||||
return {
|
||||
"new_internal_urls": new_urls,
|
||||
"missing_internal_urls": missing_urls,
|
||||
|
|
@ -235,6 +253,7 @@ def compare_snapshots(baseline_pages: dict, current_pages: dict, cfg: dict) -> d
|
|||
"removed_external_domains": sorted(old_ext - new_ext),
|
||||
"page_diffs": page_diffs,
|
||||
"new_pages_analysis": new_pages_analysis,
|
||||
"unexpected_jsonld": unexpected_jsonld,
|
||||
"new_broken_urls": new_broken,
|
||||
"known_broken_urls": known_broken,
|
||||
"total_pages_checked": len(common_urls),
|
||||
|
|
@ -305,11 +324,6 @@ def score_diff(diff: dict, cfg: dict) -> dict:
|
|||
add(sc.get("new_inline_script_suspicious", 30),
|
||||
f"{u}: Verdächtiges Inline-Script [{', '.join(s.get('patterns', []))}]")
|
||||
|
||||
for j in pd.get("new_jsonld", []):
|
||||
if j.get("unexpected"):
|
||||
add(sc.get("unexpected_jsonld", 35),
|
||||
f"{u}: Unerwartetes JSON-LD @type={j.get('type')}")
|
||||
|
||||
if pd.get("canonical_changed"):
|
||||
add(sc.get("unexpected_canonical", 35),
|
||||
f"{u}: Canonical geändert → {pd.get('new_canonical')}")
|
||||
|
|
@ -326,6 +340,14 @@ def score_diff(diff: dict, cfg: dict) -> dict:
|
|||
f"Neuer kaputte Link: {entry['url']} (HTTP {entry['status']})")
|
||||
# known_broken_urls are reported but not scored — they were already in the baseline
|
||||
|
||||
for entry in diff.get("unexpected_jsonld", []):
|
||||
add(sc.get("unexpected_jsonld", 35),
|
||||
f"{entry['url']}: Unerwartetes JSON-LD @type={entry['type']}")
|
||||
|
||||
for entry in diff.get("suspicious_filenames", []):
|
||||
add(sc.get("suspicious_filename", 60),
|
||||
f"{entry['page']}: Verdächtiger Dateiname {entry['url']}")
|
||||
|
||||
# New pages: immediate suspicious content counts too
|
||||
for npa in diff.get("new_pages_analysis", []):
|
||||
u = npa["url"]
|
||||
|
|
@ -337,8 +359,7 @@ def score_diff(diff: dict, cfg: dict) -> dict:
|
|||
add(sc.get("meta_refresh", 40), f"{u} (neu): Meta-Refresh")
|
||||
if npa.get("comment_links"):
|
||||
add(sc.get("comment_links", 25), f"{u} (neu): Kommentar-Link")
|
||||
if npa.get("unexpected_jsonld"):
|
||||
add(sc.get("unexpected_jsonld", 35), f"{u} (neu): Unerwartetes JSON-LD")
|
||||
# Unexpected JSON-LD on new pages is covered by the unexpected_jsonld channel.
|
||||
|
||||
yellow = thr.get("yellow", 20)
|
||||
red = thr.get("red", 60)
|
||||
|
|
|
|||
|
|
@ -6,9 +6,22 @@ inline script patterns, JSON-LD, and links buried in HTML comments.
|
|||
"""
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from urllib.parse import urlencode, urljoin, urlparse, parse_qsl, urlunparse
|
||||
|
||||
from bs4 import BeautifulSoup, Comment
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# Query-param keys that are pure cache-busters on asset URLs (CSS/JS/img).
|
||||
# They change on every request and must be stripped before storing/diffing.
|
||||
CACHE_PARAM_KEYS = {"v", "ver", "version", "cache", "cb", "_", "_t"}
|
||||
|
||||
|
||||
def strip_cache_params(url: str) -> str:
|
||||
"""Remove cache-buster query params (e.g. style.css?v=12345) from a URL."""
|
||||
p = urlparse(url)
|
||||
if not p.query:
|
||||
return url
|
||||
kept = [(k, v) for k, v in parse_qsl(p.query) if k.lower() not in CACHE_PARAM_KEYS]
|
||||
return urlunparse(p._replace(query=urlencode(kept)))
|
||||
|
||||
# Inline-script patterns that warrant flagging
|
||||
_SUSPICIOUS_INLINE = [
|
||||
|
|
@ -134,16 +147,19 @@ def _extract_links(soup: BeautifulSoup, base_url: str) -> dict:
|
|||
rel = " ".join(tag.get("rel", []))
|
||||
if "canonical" in rel:
|
||||
result["canonical"] = u
|
||||
u = strip_cache_params(u) # assets carry cache-busters → normalise away
|
||||
result["link_rel"].append({"url": u, "rel": rel, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("script", src=True):
|
||||
u = resolve(tag["src"])
|
||||
if u:
|
||||
u = strip_cache_params(u)
|
||||
result["script"].append({"url": u, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("img", src=True):
|
||||
u = resolve(tag["src"])
|
||||
if u:
|
||||
u = strip_cache_params(u)
|
||||
result["img"].append({"url": u, "class": classify(u)})
|
||||
|
||||
for tag in soup.find_all("iframe", src=True):
|
||||
|
|
@ -293,6 +309,8 @@ def _extract_jsonld(soup: BeautifulSoup) -> list[dict]:
|
|||
continue
|
||||
items = data if isinstance(data, list) else [data]
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue # JSON-LD must be an object; ignore stray scalars/arrays
|
||||
t = item.get("@type", "Unknown")
|
||||
results.append({"type": t, "preview": json.dumps(item, ensure_ascii=False)[:500]})
|
||||
return results
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue