diff --git a/scanner/crawler.py b/scanner/crawler.py index 1b77b20..65ea8af 100644 --- a/scanner/crawler.py +++ b/scanner/crawler.py @@ -8,6 +8,11 @@ from bs4 import BeautifulSoup logger = logging.getLogger(__name__) +_BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" +) + # Query-param keys that indicate calendar / session / tracking / cache noise. # Matched as whole keys (not prefixes) so legitimate params like ?view= or # ?value= are never silently dropped. utm_* is a known tracking prefix family. @@ -140,6 +145,17 @@ class Crawler: timeout=self.timeout, allow_redirects=True, ) + # 403 often means bot-UA blocking; retry with browser UA + if resp.status_code == 403: + try: + r2 = self.session.get( + url, timeout=self.timeout, allow_redirects=True, + headers={"User-Agent": _BROWSER_UA}, + ) + if r2.status_code != 403: + resp = r2 + except requests.exceptions.RequestException: + pass final_url = normalize_url(resp.url) or resp.url content_type = resp.headers.get("content-type", "") is_html = "text/html" in content_type and resp.status_code == 200