diff --git a/scanner/checker.py b/scanner/checker.py index ddc0129..fb53f29 100644 --- a/scanner/checker.py +++ b/scanner/checker.py @@ -16,6 +16,11 @@ import yaml logger = logging.getLogger(__name__) +_BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" +) + # Webshell / backdoor filename markers. Word boundaries are essential: # without them "mailer" matches the legitimate "PSEMailerAntispam.js" and # "gate" matches "navigate", producing false positives on every scan. @@ -138,6 +143,18 @@ def check_external_links( if resp.status_code == 405: resp = session.get(url, timeout=timeout, allow_redirects=True, stream=True) resp.close() + # 403 often means bot-blocking, not a broken link — retry with browser UA + if resp.status_code == 403: + try: + r2 = session.head(url, timeout=timeout, allow_redirects=True, + headers={"User-Agent": _BROWSER_UA}) + if r2.status_code == 405: + r2 = session.get(url, timeout=timeout, allow_redirects=True, + stream=True, headers={"User-Agent": _BROWSER_UA}) + r2.close() + resp = r2 + except requests.exceptions.RequestException: + pass # keep original 403 results[url] = {"status": resp.status_code, "final_url": resp.url, "error": None} except requests.exceptions.RequestException as exc: results[url] = {"status": None, "final_url": None, "error": str(exc)} diff --git a/tests/test_ext_links.py b/tests/test_ext_links.py index 9730f62..638fa3c 100644 --- a/tests/test_ext_links.py +++ b/tests/test_ext_links.py @@ -76,6 +76,19 @@ class TestCheckExternalLinks: assert len(results) == 3 assert inst.head.call_count == 3 + def test_403_retries_with_browser_ua_and_succeeds(self): + """Bot-blocking 403 → retry with browser UA → 200 = link is OK.""" + with patch("scanner.checker.requests.Session") as MockSession: + inst = MockSession.return_value + ok_resp = _mock_response(200, "https://example.com/") + inst.head.side_effect = [_mock_response(403), ok_resp] + inst.close = MagicMock() + + results = check_external_links(["https://example.com/"], delay=0) + + assert results["https://example.com/"]["status"] == 200 + assert inst.head.call_count == 2 + def test_empty_list_returns_empty(self): with patch("scanner.checker.requests.Session"): results = check_external_links([], delay=0)