server: respect Brave search rate limits

docs/api/rest.md

@@ -305,6 +305,7 @@ Behavior notes:
 - For `anthropic`, image attachments are sent as Messages API `image` blocks using base64 source data; text attachments are added as `text` blocks.
 - Available Sybil-managed tool calls for `openai`, `anthropic`, `xai`, and `gemini`: `web_search` and `fetch_url`. When `CHAT_CODEX_TOOL_ENABLED=true`, `codex_exec` is also available. When `CHAT_SHELL_TOOL_ENABLED=true`, `shell_exec` is also available.
 - `web_search` returns ranked results with per-result summaries/snippets. Its backend engine is selected by `CHAT_WEB_SEARCH_ENGINE`: `exa` (default), `brave` (requires `BRAVE_SEARCH_API_KEY`), or `searxng` (requires `SEARXNG_BASE_URL`; the instance must allow `format=json`).
+- Brave searches are queued and evenly paced according to the shortest window in Brave's `X-RateLimit-Policy` response header. The backend also honors `X-RateLimit-Remaining`/`X-RateLimit-Reset` and retries `429` responses up to three times with reset-aware exponential backoff; quota resets beyond the bounded retry window fail immediately.
 - `fetch_url` fetches a URL with browser-like navigation headers and returns plaintext page content (HTML converted to text server-side).
 - `codex_exec` delegates coding, shell, repository inspection, and other complex software tasks to a persistent remote Codex CLI workspace over SSH. The server runs `codex exec --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check <non-interactive wrapped prompt>` on the configured devbox inside `CHAT_CODEX_REMOTE_WORKDIR`, with SSH stdin closed.
 - `shell_exec` runs arbitrary non-interactive shell commands on the same configured devbox, starting in `CHAT_CODEX_REMOTE_WORKDIR`. It uses `bash -lc` when bash exists, otherwise `sh -lc`, closes SSH stdin, and does not run inside the Sybil server container.

docs/api/streaming-chat.md

@@ -183,6 +183,7 @@ Terminal tool-call event:
 - `openai`: Responses calls that can enter the server-managed tool loop use `store: true` so reasoning and function-call items can be passed between tool rounds.
 - `anthropic`: streamed via event stream; emits `delta` from `content_block_delta` with `text_delta`, and emits normalized `tool_call` SSE events when Anthropic `tool_use` blocks are executed. Image attachments are sent as base64 `image` blocks and text attachments are appended as `text` blocks.
 - `web_search` uses `CHAT_WEB_SEARCH_ENGINE`: `exa` (default), `brave` (requires `BRAVE_SEARCH_API_KEY`), or `searxng` (requires `SEARXNG_BASE_URL`; the instance must allow `format=json`). This only affects chat-mode tool calls, not search-mode endpoints.
+- Brave searches are queued and evenly paced according to the shortest window in Brave's `X-RateLimit-Policy` response header. The backend also honors `X-RateLimit-Remaining`/`X-RateLimit-Reset` and retries `429` responses up to three times with reset-aware exponential backoff; quota resets beyond the bounded retry window fail immediately.
 - `codex_exec` is available only when `CHAT_CODEX_TOOL_ENABLED=true`. It SSHes to `CHAT_CODEX_REMOTE_HOST`, creates/uses `CHAT_CODEX_REMOTE_WORKDIR`, and runs `codex exec --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check <non-interactive wrapped prompt>` there with SSH stdin closed. Prefer `CHAT_CODEX_SSH_KEY_PATH` with a read-only mounted private key; `CHAT_CODEX_SSH_PRIVATE_KEY_B64` is also supported.
 - `shell_exec` is available only when `CHAT_SHELL_TOOL_ENABLED=true`. It uses the same devbox SSH configuration, starts in `CHAT_CODEX_REMOTE_WORKDIR`, and runs non-interactive shell commands there with SSH stdin closed, not inside the Sybil server container.
 - `CHAT_MAX_TOOL_ROUNDS` controls how many model/tool result cycles may occur before the backend returns a tool-call limit message; default is 100.

server/src/search/brave.ts

@@ -3,6 +3,23 @@ import { env } from "../env.js";
 
 const BRAVE_WEB_SEARCH_URL = "https://api.search.brave.com/res/v1/web/search";
 const BRAVE_SEARCH_TIMEOUT_MS = 12_000;
+const DEFAULT_BRAVE_REQUEST_INTERVAL_MS = 1_000;
+const RATE_LIMIT_INTERVAL_SAFETY_RATIO = 0.05;
+const MIN_RATE_LIMIT_INTERVAL_SAFETY_MS = 2;
+const RATE_LIMIT_RESET_SAFETY_MS = 50;
+const MAX_RATE_LIMIT_RETRIES = 3;
+const MAX_RATE_LIMIT_RETRY_DELAY_MS = 8_000;
+
+type RateLimitPolicy = {
+  limit: number;
+  windowSeconds: number;
+};
+
+let requestIntervalMs = addIntervalSafety(DEFAULT_BRAVE_REQUEST_INTERVAL_MS);
+let lastRequestAtMs = 0;
+let nextRequestAtMs = 0;
+let quotaUnavailableUntilMs = 0;
+let requestQueue = Promise.resolve();
 
 export type BraveSearchOptions = {
   numResults: number;
@@ -41,6 +58,132 @@ function requireBraveSearchApiKey() {
   return env.BRAVE_SEARCH_API_KEY;
 }
 
+function sleep(milliseconds: number) {
+  return new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
+}
+
+function addIntervalSafety(intervalMs: number) {
+  return intervalMs + Math.max(MIN_RATE_LIMIT_INTERVAL_SAFETY_MS, Math.ceil(intervalMs * RATE_LIMIT_INTERVAL_SAFETY_RATIO));
+}
+
+function parseCommaSeparatedNumbers(value: string | null) {
+  if (!value) return [];
+  return value.split(",").map((part) => Number(part.trim())).map((number) => (Number.isFinite(number) ? number : null));
+}
+
+function parseRateLimitPolicy(value: string | null): RateLimitPolicy[] {
+  if (!value) return [];
+  return value.split(",").flatMap((part) => {
+    const match = part.trim().match(/^(\d+)\s*;\s*w=(\d+)$/i);
+    if (!match) return [];
+    const limit = Number(match[1]);
+    const windowSeconds = Number(match[2]);
+    return limit > 0 && windowSeconds > 0 ? [{ limit, windowSeconds }] : [];
+  });
+}
+
+function getBurstPolicyIndex(policies: RateLimitPolicy[]) {
+  if (!policies.length) return null;
+  let burstIndex = 0;
+  for (let index = 1; index < policies.length; index += 1) {
+    if (policies[index]!.windowSeconds < policies[burstIndex]!.windowSeconds) burstIndex = index;
+  }
+  return burstIndex;
+}
+
+function updateRateLimitState(headers: Headers) {
+  const policies = parseRateLimitPolicy(headers.get("x-ratelimit-policy"));
+  const burstIndex = getBurstPolicyIndex(policies);
+  if (burstIndex === null) return;
+
+  const burstPolicy = policies[burstIndex]!;
+  const learnedIntervalMs = addIntervalSafety(Math.ceil((burstPolicy.windowSeconds * 1_000) / burstPolicy.limit));
+  if (learnedIntervalMs < requestIntervalMs && lastRequestAtMs > 0) {
+    nextRequestAtMs = Math.min(nextRequestAtMs, lastRequestAtMs + learnedIntervalMs);
+  }
+  requestIntervalMs = learnedIntervalMs;
+
+  const remaining = parseCommaSeparatedNumbers(headers.get("x-ratelimit-remaining"));
+  const resetSeconds = parseCommaSeparatedNumbers(headers.get("x-ratelimit-reset"));
+  for (let index = 0; index < policies.length; index += 1) {
+    if ((remaining[index] ?? null) === null || remaining[index]! >= 1 || (resetSeconds[index] ?? 0) <= 0) continue;
+    const unavailableUntilMs = Date.now() + resetSeconds[index]! * 1_000 + RATE_LIMIT_RESET_SAFETY_MS;
+    if (index === burstIndex) {
+      nextRequestAtMs = Math.max(nextRequestAtMs, unavailableUntilMs);
+    } else {
+      quotaUnavailableUntilMs = Math.max(quotaUnavailableUntilMs, unavailableUntilMs);
+    }
+  }
+}
+
+function assertLongTermQuotaAvailable() {
+  if (quotaUnavailableUntilMs <= Date.now()) {
+    quotaUnavailableUntilMs = 0;
+    return;
+  }
+  const resetSeconds = Math.ceil((quotaUnavailableUntilMs - Date.now()) / 1_000);
+  throw new Error(`Brave Search API long-term quota is exhausted; reset is expected in ${resetSeconds} seconds.`);
+}
+
+async function waitForRateLimitSlot() {
+  const reservation = requestQueue.then(async () => {
+    while (true) {
+      assertLongTermQuotaAvailable();
+      const waitMs = nextRequestAtMs - Date.now();
+      if (waitMs <= 0) break;
+      await sleep(waitMs);
+    }
+    lastRequestAtMs = Date.now();
+    nextRequestAtMs = lastRequestAtMs + requestIntervalMs;
+  });
+  requestQueue = reservation.catch(() => undefined);
+  await reservation;
+}
+
+function get429RetryDelayMs(headers: Headers, retryNumber: number) {
+  const remaining = parseCommaSeparatedNumbers(headers.get("x-ratelimit-remaining"));
+  const resetSeconds = parseCommaSeparatedNumbers(headers.get("x-ratelimit-reset"));
+  const exhaustedResetSeconds = resetSeconds.filter((reset, index): reset is number => reset !== null && (remaining[index] ?? 0) < 1);
+  const headerDelayMs = exhaustedResetSeconds.length ? Math.max(...exhaustedResetSeconds) * 1_000 : 0;
+  const exponentialDelayMs = 2 ** retryNumber * 1_000;
+  const delayMs = Math.max(headerDelayMs + RATE_LIMIT_RESET_SAFETY_MS, exponentialDelayMs);
+  return delayMs <= MAX_RATE_LIMIT_RETRY_DELAY_MS ? delayMs : null;
+}
+
+async function fetchBrave(url: URL) {
+  const apiKey = requireBraveSearchApiKey();
+  for (let attempt = 0; attempt <= MAX_RATE_LIMIT_RETRIES; attempt += 1) {
+    await waitForRateLimitSlot();
+
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), BRAVE_SEARCH_TIMEOUT_MS);
+    let response: Response;
+    try {
+      response = await fetch(url, {
+        signal: controller.signal,
+        headers: {
+          ...buildBrowserLikeRequestHeaders("application/json"),
+          "X-Subscription-Token": apiKey,
+        },
+      });
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    updateRateLimitState(response.headers);
+    if (response.status !== 429 || attempt === MAX_RATE_LIMIT_RETRIES) return response;
+
+    const retryDelayMs = get429RetryDelayMs(response.headers, attempt);
+    await response.arrayBuffer();
+    if (retryDelayMs === null) {
+      throw new Error("Brave Search API rate limit quota is exhausted beyond the retry window.");
+    }
+    await sleep(retryDelayMs);
+  }
+
+  throw new Error("Brave Search API request failed after rate-limit retries.");
+}
+
 function normalizeDomain(input: string) {
   const trimmed = input.trim().toLowerCase();
   if (!trimmed) return null;
@@ -131,21 +274,7 @@ function mapWebResult(result: any): BraveSearchResult {
 
 export async function searchBrave(query: string, options: BraveSearchOptions): Promise<BraveSearchResponse> {
   const url = buildSearchUrl(query, options);
-  const controller = new AbortController();
+  const response = await fetchBrave(url);
-  const timeout = setTimeout(() => controller.abort(), BRAVE_SEARCH_TIMEOUT_MS);
-
-  let response: Response;
-  try {
-    response = await fetch(url, {
-      signal: controller.signal,
-      headers: {
-        ...buildBrowserLikeRequestHeaders("application/json"),
-        "X-Subscription-Token": requireBraveSearchApiKey(),
-      },
-    });
-  } finally {
-    clearTimeout(timeout);
-  }
 
   if (!response.ok) {
     await response.arrayBuffer();
@@ -166,3 +295,11 @@ export async function searchBrave(query: string, options: BraveSearchOptions): P
     results: filterResultsByDomains(results, options).slice(0, options.numResults),
   };
 }
+
+export function resetBraveRateLimitStateForTests() {
+  requestIntervalMs = addIntervalSafety(DEFAULT_BRAVE_REQUEST_INTERVAL_MS);
+  lastRequestAtMs = 0;
+  nextRequestAtMs = 0;
+  quotaUnavailableUntilMs = 0;
+  requestQueue = Promise.resolve();
+}

server/tests/brave-search.test.ts

@@ -1,12 +1,13 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 import { env } from "../src/env.js";
-import { searchBrave } from "../src/search/brave.js";
+import { resetBraveRateLimitStateForTests, searchBrave } from "../src/search/brave.js";
 
 test("searchBrave authenticates, builds filters, and normalizes web results", async () => {
   const originalFetch = globalThis.fetch;
   const originalApiKey = env.BRAVE_SEARCH_API_KEY;
   const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
+  resetBraveRateLimitStateForTests();
   env.BRAVE_SEARCH_API_KEY = "test-brave-key";
   globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
     fetchCalls.push({ input, init });
@@ -81,6 +82,7 @@ test("searchBrave authenticates, builds filters, and normalizes web results", as
 
 test("searchBrave rejects requests without an API key", async () => {
   const originalApiKey = env.BRAVE_SEARCH_API_KEY;
+  resetBraveRateLimitStateForTests();
   env.BRAVE_SEARCH_API_KEY = undefined;
   try {
     await assert.rejects(() => searchBrave("test", { numResults: 1 }), /BRAVE_SEARCH_API_KEY not set/);
@@ -92,6 +94,7 @@ test("searchBrave rejects requests without an API key", async () => {
 test("searchBrave reports non-JSON responses", async () => {
   const originalFetch = globalThis.fetch;
   const originalApiKey = env.BRAVE_SEARCH_API_KEY;
+  resetBraveRateLimitStateForTests();
   env.BRAVE_SEARCH_API_KEY = "test-brave-key";
   globalThis.fetch = (async () =>
     new Response("upstream error", {
@@ -109,3 +112,173 @@ test("searchBrave reports non-JSON responses", async () => {
     env.BRAVE_SEARCH_API_KEY = originalApiKey;
   }
 });
+
+test("searchBrave evenly paces concurrent bursts using Brave's shortest policy window", async () => {
+  const originalFetch = globalThis.fetch;
+  const originalApiKey = env.BRAVE_SEARCH_API_KEY;
+  const requestStartedAt: number[] = [];
+  resetBraveRateLimitStateForTests();
+  env.BRAVE_SEARCH_API_KEY = "test-brave-key";
+  globalThis.fetch = (async () => {
+    requestStartedAt.push(Date.now());
+    return new Response(JSON.stringify({ web: { results: [] } }), {
+      status: 200,
+      headers: {
+        "content-type": "application/json",
+        "x-ratelimit-policy": "1;w=1, 2000;w=2678400",
+        "x-ratelimit-remaining": "1, 1999",
+        "x-ratelimit-reset": "1, 2678400",
+      },
+    });
+  }) as typeof fetch;
+
+  try {
+    await Promise.all([
+      searchBrave("burst one", { numResults: 1 }),
+      searchBrave("burst two", { numResults: 1 }),
+      searchBrave("burst three", { numResults: 1 }),
+    ]);
+
+    assert.equal(requestStartedAt.length, 3);
+    assert.ok(requestStartedAt[1]! - requestStartedAt[0]! >= 1_000);
+    assert.ok(requestStartedAt[2]! - requestStartedAt[1]! >= 1_000);
+  } finally {
+    globalThis.fetch = originalFetch;
+    env.BRAVE_SEARCH_API_KEY = originalApiKey;
+    resetBraveRateLimitStateForTests();
+  }
+});
+
+test("searchBrave adapts its pacing to a 50 request-per-second Search plan", async () => {
+  const originalFetch = globalThis.fetch;
+  const originalApiKey = env.BRAVE_SEARCH_API_KEY;
+  const requestStartedAt: number[] = [];
+  resetBraveRateLimitStateForTests();
+  env.BRAVE_SEARCH_API_KEY = "test-brave-key";
+  globalThis.fetch = (async () => {
+    requestStartedAt.push(Date.now());
+    return new Response(JSON.stringify({ web: { results: [] } }), {
+      status: 200,
+      headers: {
+        "content-type": "application/json",
+        "x-ratelimit-policy": "50;w=1, 0;w=2678400",
+        "x-ratelimit-remaining": "49, 0",
+        "x-ratelimit-reset": "1, 2678400",
+      },
+    });
+  }) as typeof fetch;
+
+  try {
+    await searchBrave("learn upgraded policy", { numResults: 1 });
+    await Promise.all(Array.from({ length: 8 }, (_, index) => searchBrave(`fast burst ${index}`, { numResults: 1 })));
+
+    assert.equal(requestStartedAt.length, 9);
+    const burstStartedAt = requestStartedAt.slice(1);
+    for (let index = 1; index < burstStartedAt.length; index += 1) {
+      assert.ok(burstStartedAt[index]! - burstStartedAt[index - 1]! >= 18);
+    }
+    assert.ok(burstStartedAt.at(-1)! - burstStartedAt[0]! < 500);
+  } finally {
+    globalThis.fetch = originalFetch;
+    env.BRAVE_SEARCH_API_KEY = originalApiKey;
+    resetBraveRateLimitStateForTests();
+  }
+});
+
+test("searchBrave retries 429 responses after the burst window resets", async () => {
+  const originalFetch = globalThis.fetch;
+  const originalApiKey = env.BRAVE_SEARCH_API_KEY;
+  let fetchCount = 0;
+  resetBraveRateLimitStateForTests();
+  env.BRAVE_SEARCH_API_KEY = "test-brave-key";
+  globalThis.fetch = (async () => {
+    fetchCount += 1;
+    const rateLimitHeaders = {
+      "content-type": "application/json",
+      "x-ratelimit-policy": "1;w=1, 2000;w=2678400",
+      "x-ratelimit-remaining": fetchCount === 1 ? "0, 1999" : "1, 1998",
+      "x-ratelimit-reset": "1, 2678400",
+    };
+    if (fetchCount === 1) {
+      return new Response(JSON.stringify({ error: { detail: "Rate limit exceeded" } }), {
+        status: 429,
+        headers: rateLimitHeaders,
+      });
+    }
+    return new Response(JSON.stringify({ web: { results: [] } }), { status: 200, headers: rateLimitHeaders });
+  }) as typeof fetch;
+
+  try {
+    const startedAt = Date.now();
+    await searchBrave("retry burst", { numResults: 1 });
+    assert.equal(fetchCount, 2);
+    assert.ok(Date.now() - startedAt >= 1_000);
+  } finally {
+    globalThis.fetch = originalFetch;
+    env.BRAVE_SEARCH_API_KEY = originalApiKey;
+    resetBraveRateLimitStateForTests();
+  }
+});
+
+test("searchBrave does not wait for exhausted long-term quotas", async () => {
+  const originalFetch = globalThis.fetch;
+  const originalApiKey = env.BRAVE_SEARCH_API_KEY;
+  resetBraveRateLimitStateForTests();
+  env.BRAVE_SEARCH_API_KEY = "test-brave-key";
+  globalThis.fetch = (async () =>
+    new Response(JSON.stringify({ error: { detail: "Quota exceeded" } }), {
+      status: 429,
+      headers: {
+        "content-type": "application/json",
+        "x-ratelimit-policy": "1;w=1, 2000;w=2678400",
+        "x-ratelimit-remaining": "0, 0",
+        "x-ratelimit-reset": "1, 100000",
+      },
+    })) as typeof fetch;
+
+  try {
+    const startedAt = Date.now();
+    await assert.rejects(
+      () => searchBrave("quota exhausted", { numResults: 1 }),
+      /rate limit quota is exhausted beyond the retry window/
+    );
+    assert.ok(Date.now() - startedAt < 1_000);
+  } finally {
+    globalThis.fetch = originalFetch;
+    env.BRAVE_SEARCH_API_KEY = originalApiKey;
+    resetBraveRateLimitStateForTests();
+  }
+});
+
+test("searchBrave blocks locally after a successful request exhausts the long-term quota", async () => {
+  const originalFetch = globalThis.fetch;
+  const originalApiKey = env.BRAVE_SEARCH_API_KEY;
+  let fetchCount = 0;
+  resetBraveRateLimitStateForTests();
+  env.BRAVE_SEARCH_API_KEY = "test-brave-key";
+  globalThis.fetch = (async () => {
+    fetchCount += 1;
+    return new Response(JSON.stringify({ web: { results: [] } }), {
+      status: 200,
+      headers: {
+        "content-type": "application/json",
+        "x-ratelimit-policy": "1;w=1, 2000;w=2678400",
+        "x-ratelimit-remaining": "0, 0",
+        "x-ratelimit-reset": "1, 100000",
+      },
+    });
+  }) as typeof fetch;
+
+  try {
+    await searchBrave("last allowed query", { numResults: 1 });
+    await assert.rejects(
+      () => searchBrave("over quota query", { numResults: 1 }),
+      /long-term quota is exhausted/
+    );
+    assert.equal(fetchCount, 1);
+  } finally {
+    globalThis.fetch = originalFetch;
+    env.BRAVE_SEARCH_API_KEY = originalApiKey;
+    resetBraveRateLimitStateForTests();
+  }
+});