From 87b7d9502f23bb0b98695ebfdaaa6c9ff5803cf0 Mon Sep 17 00:00:00 2001 From: James Magahern Date: Sun, 19 Jul 2026 21:35:47 -0700 Subject: [PATCH] server: add Brave search provider --- docker-compose.example.yml | 1 + docs/api/rest.md | 2 +- docs/api/streaming-chat.md | 2 +- server/README.md | 3 +- server/src/env.ts | 11 +- server/src/llm/chat-tools.ts | 23 ++++ server/src/search/brave.ts | 168 ++++++++++++++++++++++++++++++ server/tests/brave-search.test.ts | 111 ++++++++++++++++++++ 8 files changed, 317 insertions(+), 4 deletions(-) create mode 100644 server/src/search/brave.ts create mode 100644 server/tests/brave-search.test.ts diff --git a/docker-compose.example.yml b/docker-compose.example.yml index f1deda1..cc1119b 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -17,6 +17,7 @@ services: HERMES_AGENT_API_KEY: ${HERMES_AGENT_API_KEY:-} HERMES_AGENT_MODEL: ${HERMES_AGENT_MODEL:-} EXA_API_KEY: ${EXA_API_KEY:-} + BRAVE_SEARCH_API_KEY: ${BRAVE_SEARCH_API_KEY:-} CHAT_WEB_SEARCH_ENGINE: ${CHAT_WEB_SEARCH_ENGINE:-exa} SEARXNG_BASE_URL: ${SEARXNG_BASE_URL:-} CHAT_MAX_TOOL_ROUNDS: ${CHAT_MAX_TOOL_ROUNDS:-100} diff --git a/docs/api/rest.md b/docs/api/rest.md index e67f69a..56ca3ef 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -304,7 +304,7 @@ Behavior notes: - For `openai`, Responses calls that can enter the server-managed tool loop use `store: true` so reasoning and function-call items can be passed between tool rounds. - For `anthropic`, image attachments are sent as Messages API `image` blocks using base64 source data; text attachments are added as `text` blocks. - Available Sybil-managed tool calls for `openai`, `anthropic`, `xai`, and `gemini`: `web_search` and `fetch_url`. When `CHAT_CODEX_TOOL_ENABLED=true`, `codex_exec` is also available. When `CHAT_SHELL_TOOL_ENABLED=true`, `shell_exec` is also available. -- `web_search` returns ranked results with per-result summaries/snippets. Its backend engine is selected by `CHAT_WEB_SEARCH_ENGINE` (`exa` default, or `searxng` with `SEARXNG_BASE_URL` set). SearXNG mode requires the instance to allow `format=json`. +- `web_search` returns ranked results with per-result summaries/snippets. Its backend engine is selected by `CHAT_WEB_SEARCH_ENGINE`: `exa` (default), `brave` (requires `BRAVE_SEARCH_API_KEY`), or `searxng` (requires `SEARXNG_BASE_URL`; the instance must allow `format=json`). - `fetch_url` fetches a URL with browser-like navigation headers and returns plaintext page content (HTML converted to text server-side). - `codex_exec` delegates coding, shell, repository inspection, and other complex software tasks to a persistent remote Codex CLI workspace over SSH. The server runs `codex exec --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check ` on the configured devbox inside `CHAT_CODEX_REMOTE_WORKDIR`, with SSH stdin closed. - `shell_exec` runs arbitrary non-interactive shell commands on the same configured devbox, starting in `CHAT_CODEX_REMOTE_WORKDIR`. It uses `bash -lc` when bash exists, otherwise `sh -lc`, closes SSH stdin, and does not run inside the Sybil server container. diff --git a/docs/api/streaming-chat.md b/docs/api/streaming-chat.md index 63825eb..d94de5a 100644 --- a/docs/api/streaming-chat.md +++ b/docs/api/streaming-chat.md @@ -182,7 +182,7 @@ Terminal tool-call event: - `xai` and `hermes-agent`: image attachments are sent as Chat Completions content parts; text attachments are inlined as text parts. - `openai`: Responses calls that can enter the server-managed tool loop use `store: true` so reasoning and function-call items can be passed between tool rounds. - `anthropic`: streamed via event stream; emits `delta` from `content_block_delta` with `text_delta`, and emits normalized `tool_call` SSE events when Anthropic `tool_use` blocks are executed. Image attachments are sent as base64 `image` blocks and text attachments are appended as `text` blocks. -- `web_search` uses `CHAT_WEB_SEARCH_ENGINE` (`exa` default, or `searxng` with `SEARXNG_BASE_URL` set). SearXNG mode requires the instance to allow `format=json`. This only affects chat-mode tool calls, not search-mode endpoints. +- `web_search` uses `CHAT_WEB_SEARCH_ENGINE`: `exa` (default), `brave` (requires `BRAVE_SEARCH_API_KEY`), or `searxng` (requires `SEARXNG_BASE_URL`; the instance must allow `format=json`). This only affects chat-mode tool calls, not search-mode endpoints. - `codex_exec` is available only when `CHAT_CODEX_TOOL_ENABLED=true`. It SSHes to `CHAT_CODEX_REMOTE_HOST`, creates/uses `CHAT_CODEX_REMOTE_WORKDIR`, and runs `codex exec --dangerously-bypass-approvals-and-sandbox --skip-git-repo-check ` there with SSH stdin closed. Prefer `CHAT_CODEX_SSH_KEY_PATH` with a read-only mounted private key; `CHAT_CODEX_SSH_PRIVATE_KEY_B64` is also supported. - `shell_exec` is available only when `CHAT_SHELL_TOOL_ENABLED=true`. It uses the same devbox SSH configuration, starts in `CHAT_CODEX_REMOTE_WORKDIR`, and runs non-interactive shell commands there with SSH stdin closed, not inside the Sybil server container. - `CHAT_MAX_TOOL_ROUNDS` controls how many model/tool result cycles may occur before the backend returns a tool-call limit message; default is 100. diff --git a/server/README.md b/server/README.md index ba05b08..67f156f 100644 --- a/server/README.md +++ b/server/README.md @@ -48,7 +48,8 @@ If `ADMIN_TOKEN` is not set, the server runs in open mode (dev). - `HERMES_AGENT_API_KEY` (enables the Hermes Agent provider; set to Hermes `API_SERVER_KEY`, or any non-empty value if that local server does not require auth) - `HERMES_AGENT_MODEL` (optional fallback/override model id; defaults client-side to `hermes-agent`) - `EXA_API_KEY` -- `CHAT_WEB_SEARCH_ENGINE` (`exa` by default, or `searxng` for chat tool calls only) +- `BRAVE_SEARCH_API_KEY` (required when `CHAT_WEB_SEARCH_ENGINE=brave`) +- `CHAT_WEB_SEARCH_ENGINE` (`exa` by default; `brave` and `searxng` are also supported for chat tool calls only) - `SEARXNG_BASE_URL` (required when `CHAT_WEB_SEARCH_ENGINE=searxng`; instance must allow `format=json`) - `CHAT_MAX_TOOL_ROUNDS` (`100` by default; maximum model/tool result cycles per chat completion) - `CHAT_CODEX_TOOL_ENABLED` (`false` by default; enables the `codex_exec` chat tool for managed-tool providers) diff --git a/server/src/env.ts b/server/src/env.ts index f3a9267..be586dd 100644 --- a/server/src/env.ts +++ b/server/src/env.ts @@ -24,7 +24,7 @@ const ChatWebSearchEngineSchema = z.preprocess( const trimmed = value.trim(); return trimmed ? trimmed.toLowerCase() : undefined; }, - z.enum(["exa", "searxng"]).default("exa") + z.enum(["exa", "searxng", "brave"]).default("exa") ); const BooleanFlagSchema = z.preprocess((value) => { @@ -71,6 +71,7 @@ const EnvSchema = z.object({ HERMES_AGENT_API_KEY: OptionalTrimmedStringSchema, HERMES_AGENT_MODEL: OptionalTrimmedStringSchema, EXA_API_KEY: z.string().optional(), + BRAVE_SEARCH_API_KEY: OptionalTrimmedStringSchema, // Chat-mode web_search tool configuration. Search mode remains Exa-only for now. CHAT_WEB_SEARCH_ENGINE: ChatWebSearchEngineSchema, @@ -100,6 +101,14 @@ const EnvSchema = z.object({ }); } + if (value.CHAT_WEB_SEARCH_ENGINE === "brave" && !value.BRAVE_SEARCH_API_KEY) { + ctx.addIssue({ + code: "custom", + path: ["BRAVE_SEARCH_API_KEY"], + message: "BRAVE_SEARCH_API_KEY is required when CHAT_WEB_SEARCH_ENGINE=brave", + }); + } + if ((value.CHAT_CODEX_TOOL_ENABLED || value.CHAT_SHELL_TOOL_ENABLED) && !value.CHAT_CODEX_REMOTE_HOST) { ctx.addIssue({ code: "custom", diff --git a/server/src/llm/chat-tools.ts b/server/src/llm/chat-tools.ts index 05c0edb..093cd4f 100644 --- a/server/src/llm/chat-tools.ts +++ b/server/src/llm/chat-tools.ts @@ -7,6 +7,7 @@ import { convert as htmlToText } from "html-to-text"; import { z } from "zod"; import { buildBrowserLikeNavigationHeaders } from "../browser-fetch-headers.js"; import { env } from "../env.js"; +import { searchBrave } from "../search/brave.js"; import { exaClient } from "../search/exa.js"; import { searchSearxng } from "../search/searxng.js"; import type { ChatMessage } from "./types.js"; @@ -507,11 +508,33 @@ async function runSearxngWebSearchTool(args: WebSearchArgs): Promise { + const response = await searchBrave(args.query, { + numResults: args.numResults ?? DEFAULT_WEB_RESULTS, + includeDomains: args.includeDomains, + excludeDomains: args.excludeDomains, + }); + + return { + ok: true, + searchEngine: "brave", + query: args.query, + requestId: response.requestId, + results: response.results.map((result, index) => ({ + rank: index + 1, + ...result, + })), + }; +} + async function runWebSearchTool(input: unknown): Promise { const args = WebSearchArgsSchema.parse(input); if (env.CHAT_WEB_SEARCH_ENGINE === "searxng") { return runSearxngWebSearchTool(args); } + if (env.CHAT_WEB_SEARCH_ENGINE === "brave") { + return runBraveWebSearchTool(args); + } return runExaWebSearchTool(args); } diff --git a/server/src/search/brave.ts b/server/src/search/brave.ts new file mode 100644 index 0000000..004d024 --- /dev/null +++ b/server/src/search/brave.ts @@ -0,0 +1,168 @@ +import { buildBrowserLikeRequestHeaders } from "../browser-fetch-headers.js"; +import { env } from "../env.js"; + +const BRAVE_WEB_SEARCH_URL = "https://api.search.brave.com/res/v1/web/search"; +const BRAVE_SEARCH_TIMEOUT_MS = 12_000; + +export type BraveSearchOptions = { + numResults: number; + includeDomains?: string[]; + excludeDomains?: string[]; +}; + +export type BraveSearchResult = { + title: string | null; + url: string | null; + publishedDate: string | null; + author: string | null; + summary: string | null; + text: string | null; + highlights: string[]; +}; + +export type BraveSearchResponse = { + query: string; + requestId: string | null; + results: BraveSearchResult[]; +}; + +function clipText(input: string, maxCharacters: number) { + return input.length <= maxCharacters ? input : `${input.slice(0, maxCharacters)}...`; +} + +function compactWhitespace(input: string) { + return input.replace(/\r/g, "").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").replace(/\s+/g, " ").trim(); +} + +function requireBraveSearchApiKey() { + if (!env.BRAVE_SEARCH_API_KEY) { + throw new Error("BRAVE_SEARCH_API_KEY not set"); + } + return env.BRAVE_SEARCH_API_KEY; +} + +function normalizeDomain(input: string) { + const trimmed = input.trim().toLowerCase(); + if (!trimmed) return null; + + try { + const parsed = new URL(trimmed.includes("://") ? trimmed : `https://${trimmed}`); + return parsed.hostname.replace(/^www\./, ""); + } catch { + return trimmed.split(/[/?#]/, 1)[0]?.replace(/^www\./, "") || null; + } +} + +function normalizeDomains(input: string[] | undefined) { + return Array.from(new Set((input ?? []).map(normalizeDomain).filter((domain): domain is string => Boolean(domain)))); +} + +function hostnameMatchesDomain(urlRaw: string | null, domain: string) { + if (!urlRaw) return false; + try { + const hostname = new URL(urlRaw).hostname.toLowerCase().replace(/^www\./, ""); + return hostname === domain || hostname.endsWith(`.${domain}`); + } catch { + return false; + } +} + +function filterResultsByDomains(results: BraveSearchResult[], options: BraveSearchOptions) { + const includeDomains = normalizeDomains(options.includeDomains); + const excludeDomains = normalizeDomains(options.excludeDomains); + return results.filter((result) => { + if (includeDomains.length && !includeDomains.some((domain) => hostnameMatchesDomain(result.url, domain))) return false; + if (excludeDomains.some((domain) => hostnameMatchesDomain(result.url, domain))) return false; + return true; + }); +} + +function buildBraveQuery(query: string, options: BraveSearchOptions) { + const includeDomains = normalizeDomains(options.includeDomains); + const excludeDomains = normalizeDomains(options.excludeDomains); + const includeClause = + includeDomains.length === 0 + ? "" + : includeDomains.length === 1 + ? `site:${includeDomains[0]}` + : `(${includeDomains.map((domain) => `site:${domain}`).join(" OR ")})`; + const excludeClause = excludeDomains.map((domain) => `-site:${domain}`).join(" "); + return [query, includeClause, excludeClause].filter(Boolean).join(" "); +} + +function buildSearchUrl(query: string, options: BraveSearchOptions) { + const url = new URL(BRAVE_WEB_SEARCH_URL); + url.searchParams.set("q", buildBraveQuery(query, options)); + url.searchParams.set("count", String(options.numResults)); + url.searchParams.set("safesearch", "moderate"); + url.searchParams.set("result_filter", "web"); + url.searchParams.set("text_decorations", "false"); + url.searchParams.set("extra_snippets", "true"); + return url; +} + +function stringOrNull(value: unknown) { + if (typeof value !== "string") return null; + const normalized = compactWhitespace(value); + return normalized || null; +} + +function stringArray(value: unknown) { + if (!Array.isArray(value)) return []; + return value.filter((item): item is string => typeof item === "string").map(compactWhitespace).filter(Boolean); +} + +function mapWebResult(result: any): BraveSearchResult { + const description = stringOrNull(result?.description); + const extraSnippets = stringArray(result?.extra_snippets); + const snippets = [description, ...extraSnippets].filter((snippet): snippet is string => Boolean(snippet)); + const combinedText = snippets.join("\n\n"); + + return { + title: stringOrNull(result?.title), + url: stringOrNull(result?.url), + publishedDate: stringOrNull(result?.page_age), + author: stringOrNull(result?.profile?.name) ?? stringOrNull(result?.article?.author), + summary: description ? clipText(description, 1_400) : null, + text: combinedText ? clipText(combinedText, 700) : null, + highlights: snippets.slice(0, 3).map((snippet) => clipText(snippet, 280)), + }; +} + +export async function searchBrave(query: string, options: BraveSearchOptions): Promise { + const url = buildSearchUrl(query, options); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), BRAVE_SEARCH_TIMEOUT_MS); + + let response: Response; + try { + response = await fetch(url, { + signal: controller.signal, + headers: { + ...buildBrowserLikeRequestHeaders("application/json"), + "X-Subscription-Token": requireBraveSearchApiKey(), + }, + }); + } finally { + clearTimeout(timeout); + } + + if (!response.ok) { + await response.arrayBuffer(); + throw new Error(`Brave Search API request failed with status ${response.status}.`); + } + + const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; + if (!contentType.includes("application/json")) { + await response.arrayBuffer(); + throw new Error(`Brave Search API returned ${contentType || "unknown content type"}.`); + } + + const data: any = await response.json(); + const results = Array.isArray(data?.web?.results) ? data.web.results.map(mapWebResult) : []; + return { + query, + requestId: response.headers.get("x-request-id"), + results: filterResultsByDomains(results, options).slice(0, options.numResults), + }; +} diff --git a/server/tests/brave-search.test.ts b/server/tests/brave-search.test.ts new file mode 100644 index 0000000..e0f9946 --- /dev/null +++ b/server/tests/brave-search.test.ts @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { env } from "../src/env.js"; +import { searchBrave } from "../src/search/brave.js"; + +test("searchBrave authenticates, builds filters, and normalizes web results", async () => { + const originalFetch = globalThis.fetch; + const originalApiKey = env.BRAVE_SEARCH_API_KEY; + const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = []; + env.BRAVE_SEARCH_API_KEY = "test-brave-key"; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + fetchCalls.push({ input, init }); + return new Response( + JSON.stringify({ + web: { + results: [ + { + title: " Brave result ", + url: "https://docs.example.com/article", + description: "Main\n snippet", + extra_snippets: ["Extra snippet one", "Extra snippet two"], + page_age: "2026-07-18T12:00:00Z", + profile: { name: "Example Docs" }, + }, + { + title: "Excluded result", + url: "https://blocked.example.com/article", + description: "Should be filtered", + }, + ], + }, + }), + { + status: 200, + headers: { + "content-type": "application/json; charset=utf-8", + "x-request-id": "brave-request-1", + }, + } + ); + }) as typeof fetch; + + try { + const response = await searchBrave("latest docs", { + numResults: 5, + includeDomains: ["https://example.com/path"], + excludeDomains: ["blocked.example.com"], + }); + + assert.equal(fetchCalls.length, 1); + const requestUrl = new URL(String(fetchCalls[0]?.input)); + assert.equal(requestUrl.origin + requestUrl.pathname, "https://api.search.brave.com/res/v1/web/search"); + assert.equal(requestUrl.searchParams.get("q"), "latest docs site:example.com -site:blocked.example.com"); + assert.equal(requestUrl.searchParams.get("count"), "5"); + assert.equal(requestUrl.searchParams.get("safesearch"), "moderate"); + assert.equal(requestUrl.searchParams.get("result_filter"), "web"); + assert.equal(requestUrl.searchParams.get("text_decorations"), "false"); + assert.equal(requestUrl.searchParams.get("extra_snippets"), "true"); + assert.equal((fetchCalls[0]?.init?.headers as Record)["X-Subscription-Token"], "test-brave-key"); + + assert.deepEqual(response, { + query: "latest docs", + requestId: "brave-request-1", + results: [ + { + title: "Brave result", + url: "https://docs.example.com/article", + publishedDate: "2026-07-18T12:00:00Z", + author: "Example Docs", + summary: "Main snippet", + text: "Main snippet\n\nExtra snippet one\n\nExtra snippet two", + highlights: ["Main snippet", "Extra snippet one", "Extra snippet two"], + }, + ], + }); + } finally { + globalThis.fetch = originalFetch; + env.BRAVE_SEARCH_API_KEY = originalApiKey; + } +}); + +test("searchBrave rejects requests without an API key", async () => { + const originalApiKey = env.BRAVE_SEARCH_API_KEY; + env.BRAVE_SEARCH_API_KEY = undefined; + try { + await assert.rejects(() => searchBrave("test", { numResults: 1 }), /BRAVE_SEARCH_API_KEY not set/); + } finally { + env.BRAVE_SEARCH_API_KEY = originalApiKey; + } +}); + +test("searchBrave reports non-JSON responses", async () => { + const originalFetch = globalThis.fetch; + const originalApiKey = env.BRAVE_SEARCH_API_KEY; + env.BRAVE_SEARCH_API_KEY = "test-brave-key"; + globalThis.fetch = (async () => + new Response("upstream error", { + status: 200, + headers: { "content-type": "text/plain" }, + })) as typeof fetch; + + try { + await assert.rejects( + () => searchBrave("test", { numResults: 1 }), + /Brave Search API returned text\/plain/ + ); + } finally { + globalThis.fetch = originalFetch; + env.BRAVE_SEARCH_API_KEY = originalApiKey; + } +});