From 792ca67c0828f323475e3882e35963f5e1f486f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dieter=20Schl=C3=BCter?= <dschlueter@kitux.de>
Date: Fri, 12 Jun 2026 17:17:37 +0200
Subject: [PATCH] feat: single-line progress indicator during crawl and
 ext-link check
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Uses \r to overwrite the same terminal line — no scrolling.
Crawler exposes on_progress callback; checker.check_external_links
gets an optional on_progress parameter.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 scanner/__main__.py | 29 ++++++++++++++++++++++++++++-
 scanner/checker.py  |  3 +++
 scanner/crawler.py  |  4 ++++
 3 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/scanner/__main__.py b/scanner/__main__.py
index dacc767..653995e 100644
--- a/scanner/__main__.py
+++ b/scanner/__main__.py
@@ -82,6 +82,23 @@ def _setup_logging(logs_dir: str, verbose: bool) -> None:
 
 
 # ---------------------------------------------------------------------------
+# Progress display helpers
+# ---------------------------------------------------------------------------
+
+def _progress(label: str, done: int, total: int | None = None) -> None:
+    """Overwrite the current terminal line with a progress update."""
+    if total:
+        line = f"  {label}: {done}/{total}"
+    else:
+        line = f"  {label}: {done}"
+    print(f"\r{line:<72}", end="", flush=True)
+
+
+def _progress_end() -> None:
+    """Clear the progress line after a phase completes."""
+    print(f"\r{'':<72}\r", end="", flush=True)
+
+
 # Crawl + extract helpers
 # ---------------------------------------------------------------------------
 
@@ -89,7 +106,11 @@ def _crawl_and_extract(cfg: dict) -> tuple[list[dict], list[dict]]:
     """Run crawler, extract each HTML page. Returns (pages, errors)."""
     logger = logging.getLogger(__name__)
     crawler = Crawler(cfg)
+    crawler.on_progress = lambda fetched, queued: _progress(
+        "Crawling", fetched, fetched + queued if queued else None
+    )
     raw = crawler.crawl()
+    _progress_end()
 
     pages: list[dict] = []
     for raw_page in raw["pages"]:
@@ -864,7 +885,13 @@ def _run_ext_links_core(cfg: dict, snap: dict) -> dict:
         return {"total": 0, "ok": 0, "broken": 0, "errors": 0,
                 "broken_links": [], "error_links": []}
 
-    results = check_external_links(list(ext_urls.keys()), timeout=cfg.get("request_timeout", 10))
+    total_ext = len(ext_urls)
+    results = check_external_links(
+        list(ext_urls.keys()),
+        timeout=cfg.get("request_timeout", 10),
+        on_progress=lambda done, total: _progress("Externe Links prüfen", done, total),
+    )
+    _progress_end()
     broken = {url: r for url, r in results.items() if r.get("status") and r["status"] >= 400}
     conn_errors = {url: r for url, r in results.items() if r.get("error")}
     return {
diff --git a/scanner/checker.py b/scanner/checker.py
index fb53f29..fafb45e 100644
--- a/scanner/checker.py
+++ b/scanner/checker.py
@@ -123,6 +123,7 @@ def check_external_links(
     urls: list[str],
     timeout: int = 10,
     delay: float = 0.3,
+    on_progress: object = None,
 ) -> dict[str, dict]:
     """
     Check a list of URLs for reachability via HEAD (fallback: GET).
@@ -158,6 +159,8 @@ def check_external_links(
                 results[url] = {"status": resp.status_code, "final_url": resp.url, "error": None}
             except requests.exceptions.RequestException as exc:
                 results[url] = {"status": None, "final_url": None, "error": str(exc)}
+            if on_progress:
+                on_progress(i + 1, len(urls))
     finally:
         session.close()
 
diff --git a/scanner/crawler.py b/scanner/crawler.py
index 65ea8af..55c3d89 100644
--- a/scanner/crawler.py
+++ b/scanner/crawler.py
@@ -91,6 +91,7 @@ class Crawler:
         self.headers = {"User-Agent": cfg["user_agent"]}
         self.session = requests.Session()
         self.session.headers.update(self.headers)
+        self.on_progress: object = None  # callable(fetched: int, queued: int)
 
     def crawl(self) -> dict:
         """
@@ -176,6 +177,9 @@ class Crawler:
                         if link_url not in seen and link_url not in queue:
                             queue.append(link_url)
 
+                if self.on_progress:
+                    self.on_progress(len(pages), len(queue))
+
             except requests.exceptions.RequestException as exc:
                 errors.append({"url": url, "error": str(exc)})
                 logger.warning("Error crawling %s: %s", url, exc)