fix: exclude internal comment-links; add initial external domain whitelist
- extractor.py: _find_comment_links() liefert nur noch externe Links. Interne Links in Kommentaren sind CMS-Artefakte (auskommentierte Navigations-Fragmente), keine SEO-Spam-Indikatoren. - allowed_external.yaml: 36 legitime externe Domains nach initialem Crawl von bredelar.info manuell geprüft und eingetragen. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
de90dbc19c
commit
5d52db6eed
2 changed files with 60 additions and 18 deletions
|
|
@ -303,10 +303,18 @@ def _extract_jsonld(soup: BeautifulSoup) -> list[dict]:
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _find_comment_links(html: str, base_url: str) -> list[str]:
|
||||
"""Find URLs hidden inside HTML comments."""
|
||||
"""
|
||||
Find external URLs hidden inside HTML comments.
|
||||
Internal links are excluded — CMS templates routinely comment out
|
||||
navigation fragments that link back to the same domain.
|
||||
"""
|
||||
base_netloc = urlparse(base_url).netloc
|
||||
found: list[str] = []
|
||||
for comment in re.findall(r"<!--(.*?)-->", html, re.S):
|
||||
for href in re.findall(r'href=["\']([^"\']+)["\']', comment, re.I):
|
||||
if href.startswith(("http://", "https://", "/")):
|
||||
found.append(urljoin(base_url, href))
|
||||
if not href.startswith(("http://", "https://")):
|
||||
continue
|
||||
if urlparse(href).netloc == base_netloc:
|
||||
continue # internal comment link — CMS artifact, not a threat
|
||||
found.append(href)
|
||||
return found
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue