From 9801ce6fc41016eb995ec024d0bc0b8c1c860906 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dieter=20Schl=C3=BCter?= <dschlueter@kitux.de>
Date: Fri, 12 Jun 2026 17:58:52 +0200
Subject: [PATCH] =?UTF-8?q?feat:=20sitemap=20seeding=20=E2=80=94=20discove?=
 =?UTF-8?q?r=20orphan=20pages=20not=20reachable=20via=20links?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Reads robots.txt Sitemap directive and fetches sitemap.xml / sitemap_index.xml
before crawling. All listed URLs are added to the crawl queue so pages
without any inbound links are still scanned. Configurable via crawl.sitemap.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 meine-seite.de/config.yaml |  5 +++
 scanner/config.py          |  1 +
 scanner/crawler.py         | 75 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 81 insertions(+)

diff --git a/meine-seite.de/config.yaml b/meine-seite.de/config.yaml
index e9cd9f9..baed84e 100644
--- a/meine-seite.de/config.yaml
+++ b/meine-seite.de/config.yaml
@@ -38,6 +38,11 @@ crawl:
 
   respect_robots_txt: false
 
+  # Sitemap lesen, um nicht verlinkte Seiten (Orphan Pages) zu entdecken.
+  # Der Scanner liest sitemap.xml (bzw. robots.txt → Sitemap:) und fügt alle
+  # dort aufgeführten URLs in die Crawl-Queue ein.
+  sitemap: true
+
   # Pfade, die vom Scan ausgeschlossen werden sollen.
   # Nützlich für Bereiche mit gewollt häufig wechselnden Inhalten,
   # z. B. RSS-Feeds, News-Archive, Nutzerprofile.
diff --git a/scanner/config.py b/scanner/config.py
index c45c340..9dfb54a 100644
--- a/scanner/config.py
+++ b/scanner/config.py
@@ -20,6 +20,7 @@ DEFAULT_CONFIG: dict = {
             ".mp4", ".mp3", ".webm", ".avi", ".mov",
         ],
         "exclude_paths": [],
+        "sitemap": True,
     },
     "scoring": {
         "new_external_domain": 50,
diff --git a/scanner/crawler.py b/scanner/crawler.py
index 55c3d89..2a28d7f 100644
--- a/scanner/crawler.py
+++ b/scanner/crawler.py
@@ -1,6 +1,7 @@
 """Pure-Python crawler — no wget dependency."""
 import logging
 import time
+import xml.etree.ElementTree as ET
 from urllib.parse import parse_qs, urlencode, urljoin, urlparse, urlunparse
 
 import requests
@@ -91,6 +92,7 @@ class Crawler:
         self.headers = {"User-Agent": cfg["user_agent"]}
         self.session = requests.Session()
         self.session.headers.update(self.headers)
+        self.use_sitemap: bool = cfg["crawl"].get("sitemap", True)
         self.on_progress: object = None  # callable(fetched: int, queued: int)
 
     def crawl(self) -> dict:
@@ -106,6 +108,11 @@ class Crawler:
         pages: list[dict] = []
         errors: list[dict] = []
 
+        if self.use_sitemap:
+            n = self._seed_from_sitemap(queue, seen)
+            if n:
+                logger.info("Sitemap: %d additional URL(s) added to queue", n)
+
         try:
             self._crawl_loop(queue, seen, pages, errors)
         finally:
@@ -184,6 +191,74 @@ class Crawler:
                 errors.append({"url": url, "error": str(exc)})
                 logger.warning("Error crawling %s: %s", url, exc)
 
+    def _find_sitemap_urls(self) -> list[str]:
+        """Return sitemap URLs listed in robots.txt, or empty list."""
+        urls = []
+        try:
+            resp = self.session.get(
+                f"{self.target}/robots.txt", timeout=self.timeout
+            )
+            if resp.status_code == 200:
+                for line in resp.text.splitlines():
+                    if line.lower().startswith("sitemap:"):
+                        url = line.split(":", 1)[1].strip()
+                        if url:
+                            urls.append(url)
+        except requests.exceptions.RequestException:
+            pass
+        return urls
+
+    def _seed_from_sitemap(self, queue: list[str], seen: set[str]) -> int:
+        """
+        Fetch sitemap(s) and add unseen URLs to queue.
+        Handles sitemap index files (one level of nesting).
+        Returns the number of URLs added.
+        """
+        _SM = "http://www.sitemaps.org/schemas/sitemap/0.9"
+
+        sitemap_urls = self._find_sitemap_urls()
+        if not sitemap_urls:
+            sitemap_urls = [f"{self.target}/sitemap.xml"]
+
+        visited: set[str] = set()
+        pending = list(sitemap_urls)
+        added = 0
+
+        while pending:
+            smap_url = pending.pop(0)
+            if smap_url in visited:
+                continue
+            visited.add(smap_url)
+
+            try:
+                resp = self.session.get(smap_url, timeout=self.timeout)
+                if resp.status_code != 200:
+                    continue
+                root = ET.fromstring(resp.content)
+            except Exception:
+                continue
+
+            tag = root.tag.split("}")[-1] if "}" in root.tag else root.tag
+
+            if tag == "sitemapindex":
+                # Sitemap index — collect child sitemap URLs
+                for loc in root.iter(f"{{{_SM}}}loc"):
+                    child_url = loc.text.strip() if loc.text else ""
+                    if child_url and child_url not in visited:
+                        pending.append(child_url)
+            else:
+                # Regular sitemap — collect page URLs
+                for loc in root.iter(f"{{{_SM}}}loc"):
+                    raw = loc.text.strip() if loc.text else ""
+                    url = normalize_url(raw)
+                    if url and should_crawl(
+                        url, self.base_netloc, self.skip_ext, self.exclude_paths
+                    ) and url not in seen and url not in queue:
+                        queue.append(url)
+                        added += 1
+
+        return added
+
     def _extract_follow_links(self, html: str, base_url: str) -> list[str]:
         """Extract internal <a href> links worth following."""
         result = []