From f964e4120127010103f02f30838721183aa1ef82 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 12 Jun 2026 14:46:06 +0200 Subject: [PATCH] fix: retry crawler with browser UA on 403 (bot-blocking) Some servers return 403 for the scanner's bot user-agent but serve the page normally to browsers. On a 403 response, the crawler now retries with a browser-like UA; if the server then returns 2xx, the page is crawled normally (HTML extracted, status recorded as 2xx). Mirrors the same fix already applied to check_external_links. Co-Authored-By: Claude Sonnet 4.6 --- scanner/crawler.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scanner/crawler.py b/scanner/crawler.py index 1b77b20..65ea8af 100644 --- a/scanner/crawler.py +++ b/scanner/crawler.py @@ -8,6 +8,11 @@ from bs4 import BeautifulSoup logger = logging.getLogger(__name__) +_BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" +) + # Query-param keys that indicate calendar / session / tracking / cache noise. # Matched as whole keys (not prefixes) so legitimate params like ?view= or # ?value= are never silently dropped. utm_* is a known tracking prefix family. @@ -140,6 +145,17 @@ class Crawler: timeout=self.timeout, allow_redirects=True, ) + # 403 often means bot-UA blocking; retry with browser UA + if resp.status_code == 403: + try: + r2 = self.session.get( + url, timeout=self.timeout, allow_redirects=True, + headers={"User-Agent": _BROWSER_UA}, + ) + if r2.status_code != 403: + resp = r2 + except requests.exceptions.RequestException: + pass final_url = normalize_url(resp.url) or resp.url content_type = resp.headers.get("content-type", "") is_html = "text/html" in content_type and resp.status_code == 200