feat: single-line progress indicator during crawl and ext-link check

Uses \r to overwrite the same terminal line — no scrolling.
Crawler exposes on_progress callback; checker.check_external_links
gets an optional on_progress parameter.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Dieter Schlüter 2026-06-12 17:17:37 +02:00
commit 792ca67c08
3 changed files with 35 additions and 1 deletions

View file

@ -91,6 +91,7 @@ class Crawler:
self.headers = {"User-Agent": cfg["user_agent"]}
self.session = requests.Session()
self.session.headers.update(self.headers)
self.on_progress: object = None # callable(fetched: int, queued: int)
def crawl(self) -> dict:
"""
@ -176,6 +177,9 @@ class Crawler:
if link_url not in seen and link_url not in queue:
queue.append(link_url)
if self.on_progress:
self.on_progress(len(pages), len(queue))
except requests.exceptions.RequestException as exc:
errors.append({"url": url, "error": str(exc)})
logger.warning("Error crawling %s: %s", url, exc)