fix most web_fetches from getting blocked using a real user agent

This commit is contained in:
2026-06-11 23:36:19 -07:00
parent 22aa652257
commit d7214c88ad
8 changed files with 403 additions and 54 deletions

View File

@@ -1,3 +1,4 @@
import { buildBrowserLikeRequestHeaders } from "../browser-fetch-headers.js";
import { env } from "../env.js";
const SEARXNG_TIMEOUT_MS = 12_000;
@@ -106,10 +107,7 @@ async function fetchSearxng(url: URL, accept: string) {
return await fetch(url, {
redirect: "follow",
signal: controller.signal,
headers: {
"User-Agent": "SybilBot/1.0 (+https://sybil.local)",
Accept: accept,
},
headers: buildBrowserLikeRequestHeaders(accept),
});
} finally {
clearTimeout(timeout);