feat: sitemap seeding — discover orphan pages not reachable via links

Reads robots.txt Sitemap directive and fetches sitemap.xml / sitemap_index.xml
before crawling. All listed URLs are added to the crawl queue so pages
without any inbound links are still scanned. Configurable via crawl.sitemap.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scanner/config.py

@@ -20,6 +20,7 @@ DEFAULT_CONFIG: dict = {
             ".mp4", ".mp3", ".webm", ".avi", ".mov",
         ],
         "exclude_paths": [],
+        "sitemap": True,
     },
     "scoring": {
         "new_external_domain": 50,

scanner/crawler.py

@@ -1,6 +1,7 @@
 """Pure-Python crawler — no wget dependency."""
 import logging
 import time
+import xml.etree.ElementTree as ET
 from urllib.parse import parse_qs, urlencode, urljoin, urlparse, urlunparse
 
 import requests
@@ -91,6 +92,7 @@ class Crawler:
         self.headers = {"User-Agent": cfg["user_agent"]}
         self.session = requests.Session()
         self.session.headers.update(self.headers)
+        self.use_sitemap: bool = cfg["crawl"].get("sitemap", True)
         self.on_progress: object = None  # callable(fetched: int, queued: int)
 
     def crawl(self) -> dict:
@@ -106,6 +108,11 @@ class Crawler:
         pages: list[dict] = []
         errors: list[dict] = []
 
+        if self.use_sitemap:
+            n = self._seed_from_sitemap(queue, seen)
+            if n:
+                logger.info("Sitemap: %d additional URL(s) added to queue", n)
+
         try:
             self._crawl_loop(queue, seen, pages, errors)
         finally:
@@ -184,6 +191,74 @@ class Crawler:
                 errors.append({"url": url, "error": str(exc)})
                 logger.warning("Error crawling %s: %s", url, exc)
 
+    def _find_sitemap_urls(self) -> list[str]:
+        """Return sitemap URLs listed in robots.txt, or empty list."""
+        urls = []
+        try:
+            resp = self.session.get(
+                f"{self.target}/robots.txt", timeout=self.timeout
+            )
+            if resp.status_code == 200:
+                for line in resp.text.splitlines():
+                    if line.lower().startswith("sitemap:"):
+                        url = line.split(":", 1)[1].strip()
+                        if url:
+                            urls.append(url)
+        except requests.exceptions.RequestException:
+            pass
+        return urls
+
+    def _seed_from_sitemap(self, queue: list[str], seen: set[str]) -> int:
+        """
+        Fetch sitemap(s) and add unseen URLs to queue.
+        Handles sitemap index files (one level of nesting).
+        Returns the number of URLs added.
+        """
+        _SM = "http://www.sitemaps.org/schemas/sitemap/0.9"
+
+        sitemap_urls = self._find_sitemap_urls()
+        if not sitemap_urls:
+            sitemap_urls = [f"{self.target}/sitemap.xml"]
+
+        visited: set[str] = set()
+        pending = list(sitemap_urls)
+        added = 0
+
+        while pending:
+            smap_url = pending.pop(0)
+            if smap_url in visited:
+                continue
+            visited.add(smap_url)
+
+            try:
+                resp = self.session.get(smap_url, timeout=self.timeout)
+                if resp.status_code != 200:
+                    continue
+                root = ET.fromstring(resp.content)
+            except Exception:
+                continue
+
+            tag = root.tag.split("}")[-1] if "}" in root.tag else root.tag
+
+            if tag == "sitemapindex":
+                # Sitemap index — collect child sitemap URLs
+                for loc in root.iter(f"{{{_SM}}}loc"):
+                    child_url = loc.text.strip() if loc.text else ""
+                    if child_url and child_url not in visited:
+                        pending.append(child_url)
+            else:
+                # Regular sitemap — collect page URLs
+                for loc in root.iter(f"{{{_SM}}}loc"):
+                    raw = loc.text.strip() if loc.text else ""
+                    url = normalize_url(raw)
+                    if url and should_crawl(
+                        url, self.base_netloc, self.skip_ext, self.exclude_paths
+                    ) and url not in seen and url not in queue:
+                        queue.append(url)
+                        added += 1
+
+        return added
+
     def _extract_follow_links(self, html: str, base_url: str) -> list[str]:
         """Extract internal <a href> links worth following."""
         result = []