de90dbc19c
PyYAML parst auskommentierte Listen als None statt []. Da der Key vorhanden ist, greift .get(key, []) nicht. Drei Fixes: - extractor.py, differ.py: `or []` statt Default-Argument - config.py: _deep_update überspringt None-Override auf List-Defaults Tritt auf bei `ignore_selectors` und `ignore_patterns` in config.yaml. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
312 lines
11 KiB
Python
312 lines
11 KiB
Python
"""
|
|
HTML extraction with BeautifulSoup4+lxml.
|
|
|
|
Extracts: visible text, links (all types), metadata, hidden-content indicators,
|
|
inline script patterns, JSON-LD, and links buried in HTML comments.
|
|
"""
|
|
import json
|
|
import re
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
from bs4 import BeautifulSoup, Comment
|
|
|
|
# Inline-script patterns that warrant flagging
|
|
_SUSPICIOUS_INLINE = [
|
|
(r"eval\s*\(", "eval()"),
|
|
(r"atob\s*\(", "atob()"),
|
|
(r"fromCharCode", "fromCharCode"),
|
|
(r"document\.createElement\s*\(\s*[\"']script[\"']", "createElement(script)"),
|
|
(r"\bfetch\s*\(", "fetch()"),
|
|
(r"XMLHttpRequest", "XMLHttpRequest"),
|
|
(r"base64_decode", "base64_decode"),
|
|
]
|
|
|
|
# CSS-based hiding patterns (applied to inline style attributes)
|
|
_HIDDEN_STYLE_CHECKS = [
|
|
(r"display\s*:\s*none", "display:none"),
|
|
(r"visibility\s*:\s*hidden", "visibility:hidden"),
|
|
(r"opacity\s*:\s*0(?:\.0+)?(?!\d)", "opacity:0"),
|
|
(r"font-size\s*:\s*0", "font-size:0"),
|
|
(r"color\s*:\s*(?:white|#fff(?:fff)?|rgba?\(\s*255\s*,\s*255\s*,\s*255)", "white-text"),
|
|
(r"width\s*:\s*0(?:px)?[;\s]", "width:0"),
|
|
(r"height\s*:\s*0(?:px)?[;\s]", "height:0"),
|
|
(r"position\s*:\s*absolute", "position:absolute"), # combined with off-screen check below
|
|
(r"(?:left|top)\s*:\s*-\d{3,}", "off-screen-position"),
|
|
(r"clip\s*:\s*rect\s*\(\s*0\s*,?\s*0\s*,?\s*0\s*,?\s*0\s*\)", "clip:rect(0)"),
|
|
(r"text-indent\s*:\s*-\d{3,}", "text-indent-offscreen"),
|
|
]
|
|
|
|
|
|
def extract_page(html: str, url: str, cfg: dict | None = None) -> dict:
|
|
"""
|
|
Full extraction from page HTML.
|
|
|
|
Returns a structured dict with all data needed for diffing and checking.
|
|
"""
|
|
cfg = cfg or {}
|
|
soup = BeautifulSoup(html, "lxml")
|
|
|
|
# Remove ignored selectors before text extraction
|
|
ignore_selectors = cfg.get("normalization", {}).get("ignore_selectors") or []
|
|
for sel in ignore_selectors:
|
|
for el in soup.select(sel):
|
|
el.decompose()
|
|
|
|
return {
|
|
"url": url,
|
|
"text": _visible_text(soup),
|
|
"links": _extract_links(soup, url),
|
|
"metadata": _extract_metadata(soup),
|
|
"hidden_content": _find_hidden_content(soup),
|
|
"inline_scripts": _check_inline_scripts(soup),
|
|
"jsonld": _extract_jsonld(soup),
|
|
"comment_links": _find_comment_links(html, url),
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Text
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _visible_text(soup: BeautifulSoup) -> str:
|
|
"""Extract visible text — strips scripts, styles, noscript, head."""
|
|
work = soup.__copy__()
|
|
for tag in work(["script", "style", "noscript", "head", "meta", "link"]):
|
|
tag.decompose()
|
|
text = work.get_text(separator=" ")
|
|
return re.sub(r"\s+", " ", text).strip()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Links
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _extract_links(soup: BeautifulSoup, base_url: str) -> dict:
|
|
"""
|
|
Extract all link categories:
|
|
a, link, script[src], img[src], iframe[src], form[action],
|
|
meta_refresh, canonical
|
|
Each entry: {url, class: 'internal'|'external', ...extra}
|
|
"""
|
|
base_netloc = urlparse(base_url).netloc
|
|
|
|
def classify(u: str) -> str:
|
|
return "internal" if urlparse(u).netloc == base_netloc else "external"
|
|
|
|
def resolve(href: str | None) -> str | None:
|
|
if not href:
|
|
return None
|
|
href = href.strip()
|
|
if href.startswith(("javascript:", "mailto:", "tel:", "data:", "#")):
|
|
return None
|
|
abs_url = urljoin(base_url, href)
|
|
p = urlparse(abs_url)
|
|
if p.scheme not in ("http", "https"):
|
|
return None
|
|
# Drop fragment
|
|
return abs_url.split("#")[0]
|
|
|
|
result: dict = {
|
|
"a": [],
|
|
"link_rel": [],
|
|
"script": [],
|
|
"img": [],
|
|
"iframe": [],
|
|
"form": [],
|
|
"meta_refresh": [],
|
|
"canonical": None,
|
|
}
|
|
|
|
for tag in soup.find_all("a", href=True):
|
|
u = resolve(tag["href"])
|
|
if u:
|
|
result["a"].append({
|
|
"url": u,
|
|
"text": tag.get_text(" ", strip=True)[:200],
|
|
"class": classify(u),
|
|
"rel": " ".join(tag.get("rel", [])),
|
|
})
|
|
|
|
for tag in soup.find_all("link", href=True):
|
|
u = resolve(tag["href"])
|
|
if not u:
|
|
continue
|
|
rel = " ".join(tag.get("rel", []))
|
|
if "canonical" in rel:
|
|
result["canonical"] = u
|
|
result["link_rel"].append({"url": u, "rel": rel, "class": classify(u)})
|
|
|
|
for tag in soup.find_all("script", src=True):
|
|
u = resolve(tag["src"])
|
|
if u:
|
|
result["script"].append({"url": u, "class": classify(u)})
|
|
|
|
for tag in soup.find_all("img", src=True):
|
|
u = resolve(tag["src"])
|
|
if u:
|
|
result["img"].append({"url": u, "class": classify(u)})
|
|
|
|
for tag in soup.find_all("iframe", src=True):
|
|
u = resolve(tag["src"])
|
|
if u:
|
|
result["iframe"].append({"url": u, "class": classify(u)})
|
|
|
|
for tag in soup.find_all("form", action=True):
|
|
u = resolve(tag["action"])
|
|
if u:
|
|
result["form"].append({"url": u, "class": classify(u)})
|
|
|
|
for tag in soup.find_all("meta"):
|
|
if tag.get("http-equiv", "").lower() == "refresh":
|
|
content = tag.get("content", "")
|
|
m = re.search(r"url\s*=\s*(.+)", content, re.I)
|
|
if m:
|
|
u = resolve(m.group(1).strip().strip("\"'"))
|
|
if u:
|
|
result["meta_refresh"].append(u)
|
|
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Metadata
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _extract_metadata(soup: BeautifulSoup) -> dict:
|
|
"""Extract title, description, canonical, og-tags, robots, hreflang, H1/H2."""
|
|
meta: dict = {}
|
|
|
|
t = soup.find("title")
|
|
meta["title"] = t.get_text(strip=True) if t else None
|
|
|
|
canonical_tag = soup.find("link", rel="canonical")
|
|
if canonical_tag:
|
|
meta["canonical"] = canonical_tag.get("href", "").strip()
|
|
|
|
for tag in soup.find_all("meta"):
|
|
name = (tag.get("name") or "").lower().strip()
|
|
prop = (tag.get("property") or "").lower().strip()
|
|
content = tag.get("content", "")
|
|
|
|
if name == "description":
|
|
meta["description"] = content
|
|
elif name == "robots":
|
|
meta["robots"] = content
|
|
elif name == "keywords":
|
|
meta["keywords"] = content
|
|
elif prop.startswith("og:"):
|
|
meta[prop] = content
|
|
elif prop.startswith("twitter:"):
|
|
meta[prop] = content
|
|
|
|
hreflang = []
|
|
for tag in soup.find_all("link", rel="alternate"):
|
|
hl = tag.get("hreflang")
|
|
if hl:
|
|
hreflang.append({"lang": hl, "href": tag.get("href", "")})
|
|
meta["hreflang"] = hreflang
|
|
|
|
meta["h1"] = [h.get_text(" ", strip=True) for h in soup.find_all("h1")]
|
|
meta["h2"] = [h.get_text(" ", strip=True) for h in soup.find_all("h2")]
|
|
|
|
return meta
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Hidden content
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _find_hidden_content(soup: BeautifulSoup) -> list[dict]:
|
|
"""
|
|
Detect elements that are visually hidden but contain text or links.
|
|
Catches common SEO-spam hiding techniques.
|
|
"""
|
|
findings: list[dict] = []
|
|
|
|
for tag in soup.find_all(style=True):
|
|
style = tag.get("style", "")
|
|
matched_indicators = []
|
|
for pattern, label in _HIDDEN_STYLE_CHECKS:
|
|
if re.search(pattern, style, re.I):
|
|
matched_indicators.append(label)
|
|
if not matched_indicators:
|
|
continue
|
|
|
|
text = tag.get_text(" ", strip=True)[:300]
|
|
links = [a.get("href", "") for a in tag.find_all("a", href=True)]
|
|
if text or links:
|
|
findings.append({
|
|
"type": "inline_style_hidden",
|
|
"indicators": matched_indicators,
|
|
"tag": tag.name,
|
|
"text_preview": text,
|
|
"links": links,
|
|
})
|
|
|
|
# <noscript> with links
|
|
for tag in soup.find_all("noscript"):
|
|
links = re.findall(r'href=["\']([^"\']+)["\']', str(tag), re.I)
|
|
if links:
|
|
findings.append({
|
|
"type": "noscript_links",
|
|
"links": links,
|
|
"text_preview": tag.get_text(" ", strip=True)[:200],
|
|
})
|
|
|
|
return findings
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Inline scripts
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _check_inline_scripts(soup: BeautifulSoup) -> list[dict]:
|
|
"""Find inline <script> blocks that match suspicious patterns."""
|
|
findings: list[dict] = []
|
|
for i, tag in enumerate(soup.find_all("script"), 1):
|
|
if tag.get("src"):
|
|
continue
|
|
code = tag.get_text() or ""
|
|
hits = [label for pat, label in _SUSPICIOUS_INLINE if re.search(pat, code, re.I)]
|
|
if hits:
|
|
findings.append({
|
|
"script_index": i,
|
|
"patterns": hits,
|
|
"preview": code[:400],
|
|
})
|
|
return findings
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# JSON-LD
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _extract_jsonld(soup: BeautifulSoup) -> list[dict]:
|
|
"""Extract JSON-LD blocks and flag unexpected @types."""
|
|
results: list[dict] = []
|
|
for tag in soup.find_all("script", type="application/ld+json"):
|
|
raw = tag.string or ""
|
|
try:
|
|
data = json.loads(raw)
|
|
except (json.JSONDecodeError, ValueError):
|
|
results.append({"type": "PARSE_ERROR", "unexpected": True, "preview": raw[:200]})
|
|
continue
|
|
items = data if isinstance(data, list) else [data]
|
|
for item in items:
|
|
t = item.get("@type", "Unknown")
|
|
results.append({"type": t, "preview": json.dumps(item, ensure_ascii=False)[:500]})
|
|
return results
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Comment links
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _find_comment_links(html: str, base_url: str) -> list[str]:
|
|
"""Find URLs hidden inside HTML comments."""
|
|
found: list[str] = []
|
|
for comment in re.findall(r"<!--(.*?)-->", html, re.S):
|
|
for href in re.findall(r'href=["\']([^"\']+)["\']', comment, re.I):
|
|
if href.startswith(("http://", "https://", "/")):
|
|
found.append(urljoin(base_url, href))
|
|
return found
|