fix: exclude internal comment-links; add initial external domain whitelist

- extractor.py: _find_comment_links() liefert nur noch externe Links.
  Interne Links in Kommentaren sind CMS-Artefakte (auskommentierte
  Navigations-Fragmente), keine SEO-Spam-Indikatoren.
- allowed_external.yaml: 36 legitime externe Domains nach initialem
  Crawl von bredelar.info manuell geprüft und eingetragen.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name 2026-06-12 01:15:58 +02:00
commit 5d52db6eed

View file

@ -303,10 +303,18 @@ def _extract_jsonld(soup: BeautifulSoup) -> list[dict]:
# ---------------------------------------------------------------------------
def _find_comment_links(html: str, base_url: str) -> list[str]:
"""Find URLs hidden inside HTML comments."""
"""
Find external URLs hidden inside HTML comments.
Internal links are excluded — CMS templates routinely comment out
navigation fragments that link back to the same domain.
"""
base_netloc = urlparse(base_url).netloc
found: list[str] = []
for comment in re.findall(r"<!--(.*?)-->", html, re.S):
for href in re.findall(r'href=["\']([^"\']+)["\']', comment, re.I):
if href.startswith(("http://", "https://", "/")):
found.append(urljoin(base_url, href))
if not href.startswith(("http://", "https://")):
continue
if urlparse(href).netloc == base_netloc:
continue # internal comment link — CMS artifact, not a threat
found.append(href)
return found