feat: single-line progress indicator during crawl and ext-link check
Uses \r to overwrite the same terminal line — no scrolling. Crawler exposes on_progress callback; checker.check_external_links gets an optional on_progress parameter. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
fcbccce422
commit
792ca67c08
3 changed files with 35 additions and 1 deletions
|
|
@ -91,6 +91,7 @@ class Crawler:
|
|||
self.headers = {"User-Agent": cfg["user_agent"]}
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(self.headers)
|
||||
self.on_progress: object = None # callable(fetched: int, queued: int)
|
||||
|
||||
def crawl(self) -> dict:
|
||||
"""
|
||||
|
|
@ -176,6 +177,9 @@ class Crawler:
|
|||
if link_url not in seen and link_url not in queue:
|
||||
queue.append(link_url)
|
||||
|
||||
if self.on_progress:
|
||||
self.on_progress(len(pages), len(queue))
|
||||
|
||||
except requests.exceptions.RequestException as exc:
|
||||
errors.append({"url": url, "error": str(exc)})
|
||||
logger.warning("Error crawling %s: %s", url, exc)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue