fix: retry external link check with browser UA on 403

Servers that block bot user-agents return 403 even for live URLs.
On a 403 response, the checker now retries with a browser-like UA;
if the server then responds with 2xx/3xx, the link is counted as OK.
This eliminates false positives from bot-blocking (e.g. bergbauspuren-bredelar.de).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

scanner/checker.py

@@ -16,6 +16,11 @@ import yaml
 
 logger = logging.getLogger(__name__)
 
+_BROWSER_UA = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+)
+
 # Webshell / backdoor filename markers. Word boundaries are essential:
 # without them "mailer" matches the legitimate "PSEMailerAntispam.js" and
 # "gate" matches "navigate", producing false positives on every scan.
@@ -138,6 +143,18 @@ def check_external_links(
                 if resp.status_code == 405:
                     resp = session.get(url, timeout=timeout, allow_redirects=True, stream=True)
                     resp.close()
+                # 403 often means bot-blocking, not a broken link — retry with browser UA
+                if resp.status_code == 403:
+                    try:
+                        r2 = session.head(url, timeout=timeout, allow_redirects=True,
+                                          headers={"User-Agent": _BROWSER_UA})
+                        if r2.status_code == 405:
+                            r2 = session.get(url, timeout=timeout, allow_redirects=True,
+                                             stream=True, headers={"User-Agent": _BROWSER_UA})
+                            r2.close()
+                        resp = r2
+                    except requests.exceptions.RequestException:
+                        pass  # keep original 403
                 results[url] = {"status": resp.status_code, "final_url": resp.url, "error": None}
             except requests.exceptions.RequestException as exc:
                 results[url] = {"status": None, "final_url": None, "error": str(exc)}

tests/test_ext_links.py

@@ -76,6 +76,19 @@ class TestCheckExternalLinks:
         assert len(results) == 3
         assert inst.head.call_count == 3
 
+    def test_403_retries_with_browser_ua_and_succeeds(self):
+        """Bot-blocking 403 → retry with browser UA → 200 = link is OK."""
+        with patch("scanner.checker.requests.Session") as MockSession:
+            inst = MockSession.return_value
+            ok_resp = _mock_response(200, "https://example.com/")
+            inst.head.side_effect = [_mock_response(403), ok_resp]
+            inst.close = MagicMock()
+
+            results = check_external_links(["https://example.com/"], delay=0)
+
+        assert results["https://example.com/"]["status"] == 200
+        assert inst.head.call_count == 2
+
     def test_empty_list_returns_empty(self):
         with patch("scanner.checker.requests.Session"):
             results = check_external_links([], delay=0)