From 22ed1570a451d42e45b390c35252a798ff84daed Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 12 Jun 2026 11:40:27 +0200 Subject: [PATCH] fix: retry external link check with browser UA on 403 Servers that block bot user-agents return 403 even for live URLs. On a 403 response, the checker now retries with a browser-like UA; if the server then responds with 2xx/3xx, the link is counted as OK. This eliminates false positives from bot-blocking (e.g. bergbauspuren-bredelar.de). Co-Authored-By: Claude Sonnet 4.6 --- scanner/checker.py | 17 +++++++++++++++++ tests/test_ext_links.py | 13 +++++++++++++ 2 files changed, 30 insertions(+) diff --git a/scanner/checker.py b/scanner/checker.py index ddc0129..fb53f29 100644 --- a/scanner/checker.py +++ b/scanner/checker.py @@ -16,6 +16,11 @@ import yaml logger = logging.getLogger(__name__) +_BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" +) + # Webshell / backdoor filename markers. Word boundaries are essential: # without them "mailer" matches the legitimate "PSEMailerAntispam.js" and # "gate" matches "navigate", producing false positives on every scan. @@ -138,6 +143,18 @@ def check_external_links( if resp.status_code == 405: resp = session.get(url, timeout=timeout, allow_redirects=True, stream=True) resp.close() + # 403 often means bot-blocking, not a broken link — retry with browser UA + if resp.status_code == 403: + try: + r2 = session.head(url, timeout=timeout, allow_redirects=True, + headers={"User-Agent": _BROWSER_UA}) + if r2.status_code == 405: + r2 = session.get(url, timeout=timeout, allow_redirects=True, + stream=True, headers={"User-Agent": _BROWSER_UA}) + r2.close() + resp = r2 + except requests.exceptions.RequestException: + pass # keep original 403 results[url] = {"status": resp.status_code, "final_url": resp.url, "error": None} except requests.exceptions.RequestException as exc: results[url] = {"status": None, "final_url": None, "error": str(exc)} diff --git a/tests/test_ext_links.py b/tests/test_ext_links.py index 9730f62..638fa3c 100644 --- a/tests/test_ext_links.py +++ b/tests/test_ext_links.py @@ -76,6 +76,19 @@ class TestCheckExternalLinks: assert len(results) == 3 assert inst.head.call_count == 3 + def test_403_retries_with_browser_ua_and_succeeds(self): + """Bot-blocking 403 → retry with browser UA → 200 = link is OK.""" + with patch("scanner.checker.requests.Session") as MockSession: + inst = MockSession.return_value + ok_resp = _mock_response(200, "https://example.com/") + inst.head.side_effect = [_mock_response(403), ok_resp] + inst.close = MagicMock() + + results = check_external_links(["https://example.com/"], delay=0) + + assert results["https://example.com/"]["status"] == 200 + assert inst.head.call_count == 2 + def test_empty_list_returns_empty(self): with patch("scanner.checker.requests.Session"): results = check_external_links([], delay=0)