From d7967eaa75c90489cc65b8ac46201403f6e196c5 Mon Sep 17 00:00:00 2001 From: James Magahern Date: Sat, 2 May 2026 18:14:41 -0700 Subject: [PATCH] Adds searxng support for tool calling --- docker-compose.example.yml | 2 + docs/api/rest.md | 3 +- docs/api/streaming-chat.md | 1 + server/README.md | 2 + server/src/env.ts | 33 +++++++- server/src/llm/chat-tools.ts | 42 ++++++++- server/src/search/searxng.ts | 160 +++++++++++++++++++++++++++++++++++ 7 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 server/src/search/searxng.ts diff --git a/docker-compose.example.yml b/docker-compose.example.yml index a8db2b5..b3fa4d8 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -13,6 +13,8 @@ services: ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} XAI_API_KEY: ${XAI_API_KEY:-} EXA_API_KEY: ${EXA_API_KEY:-} + CHAT_WEB_SEARCH_ENGINE: ${CHAT_WEB_SEARCH_ENGINE:-exa} + SEARXNG_BASE_URL: ${SEARXNG_BASE_URL:-} volumes: - sybil_data:/data expose: diff --git a/docs/api/rest.md b/docs/api/rest.md index b1922a6..42ca7ab 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -114,7 +114,7 @@ Behavior notes: - Server updates chat-level model metadata on each call: `lastUsedProvider`/`lastUsedModel`; first successful/failed call also initializes `initiatedProvider`/`initiatedModel` if unset. - For `openai` and `xai`, backend enables tool use during chat completion with an internal system instruction. - Available tool calls for chat: `web_search` and `fetch_url`. -- `web_search` uses Exa and returns ranked results with per-result summaries/snippets. +- `web_search` returns ranked results with per-result summaries/snippets. Its backend engine is selected by `CHAT_WEB_SEARCH_ENGINE` (`exa` default, or `searxng` with `SEARXNG_BASE_URL` set). SearXNG mode requires the instance to allow `format=json`. - `fetch_url` fetches a URL and returns plaintext page content (HTML converted to text server-side). - When a tool call is executed, backend stores a chat `Message` with `role: "tool"` and tool metadata (`metadata.kind = "tool_call"`), then stores the assistant output. - `anthropic` currently runs without server-managed tool calls. @@ -161,6 +161,7 @@ Behavior notes: Search run notes: - Backend executes Exa search and Exa answer. +- Search mode is independent from chat `web_search` tool configuration and remains Exa-only. - Persists answer text/citations + ranked results. - If both search and answer fail, endpoint returns an error. diff --git a/docs/api/streaming-chat.md b/docs/api/streaming-chat.md index 55cf17c..e5e8826 100644 --- a/docs/api/streaming-chat.md +++ b/docs/api/streaming-chat.md @@ -105,6 +105,7 @@ Event order: - `openai`: backend may execute internal tool calls (`web_search`, `fetch_url`) before producing final text. - `xai`: same tool-enabled behavior as OpenAI. - `anthropic`: streamed via event stream; emits `delta` from `content_block_delta` with `text_delta`. +- `web_search` uses `CHAT_WEB_SEARCH_ENGINE` (`exa` default, or `searxng` with `SEARXNG_BASE_URL` set). SearXNG mode requires the instance to allow `format=json`. This only affects chat-mode tool calls, not search-mode endpoints. Tool-enabled streaming notes (`openai`/`xai`): - Stream still emits standard `meta`, `delta`, `done|error` events. diff --git a/server/README.md b/server/README.md index 274219b..6422958 100644 --- a/server/README.md +++ b/server/README.md @@ -44,6 +44,8 @@ If `ADMIN_TOKEN` is not set, the server runs in open mode (dev). - `ANTHROPIC_API_KEY` - `XAI_API_KEY` - `EXA_API_KEY` +- `CHAT_WEB_SEARCH_ENGINE` (`exa` by default, or `searxng` for chat tool calls only) +- `SEARXNG_BASE_URL` (required when `CHAT_WEB_SEARCH_ENGINE=searxng`; instance must allow `format=json`) ## API - `GET /health` diff --git a/server/src/env.ts b/server/src/env.ts index e56e9e8..3fe7d8f 100644 --- a/server/src/env.ts +++ b/server/src/env.ts @@ -1,5 +1,24 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { config as loadDotenv } from "dotenv"; import { z } from "zod"; -import "dotenv/config"; + +loadDotenv({ quiet: true }); +loadDotenv({ path: path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../.env"), quiet: true }); + +const OptionalUrlSchema = z.preprocess( + (value) => (typeof value === "string" && value.trim() === "" ? undefined : value), + z.string().trim().url().optional() +); + +const ChatWebSearchEngineSchema = z.preprocess( + (value) => { + if (typeof value !== "string") return value; + const trimmed = value.trim(); + return trimmed ? trimmed.toLowerCase() : undefined; + }, + z.enum(["exa", "searxng"]).default("exa") +); const EnvSchema = z.object({ PORT: z.coerce.number().int().positive().default(8787), @@ -13,6 +32,18 @@ const EnvSchema = z.object({ ANTHROPIC_API_KEY: z.string().optional(), XAI_API_KEY: z.string().optional(), EXA_API_KEY: z.string().optional(), + + // Chat-mode web_search tool configuration. Search mode remains Exa-only for now. + CHAT_WEB_SEARCH_ENGINE: ChatWebSearchEngineSchema, + SEARXNG_BASE_URL: OptionalUrlSchema, +}).superRefine((value, ctx) => { + if (value.CHAT_WEB_SEARCH_ENGINE === "searxng" && !value.SEARXNG_BASE_URL) { + ctx.addIssue({ + code: "custom", + path: ["SEARXNG_BASE_URL"], + message: "SEARXNG_BASE_URL is required when CHAT_WEB_SEARCH_ENGINE=searxng", + }); + } }); export type Env = z.infer; diff --git a/server/src/llm/chat-tools.ts b/server/src/llm/chat-tools.ts index 91ae8c9..d96dd3c 100644 --- a/server/src/llm/chat-tools.ts +++ b/server/src/llm/chat-tools.ts @@ -1,7 +1,9 @@ import { convert as htmlToText } from "html-to-text"; import type OpenAI from "openai"; import { z } from "zod"; +import { env } from "../env.js"; import { exaClient } from "../search/exa.js"; +import { searchSearxng } from "../search/searxng.js"; import type { ChatMessage } from "./types.js"; const MAX_TOOL_ROUNDS = 4; @@ -21,6 +23,8 @@ const WebSearchArgsSchema = z }) .strict(); +type WebSearchArgs = z.infer; + const FetchUrlArgsSchema = z .object({ url: z.string().trim().url(), @@ -267,8 +271,7 @@ function normalizeIncomingMessages(messages: ChatMessage[]) { return [{ role: "system", content: CHAT_TOOL_SYSTEM_PROMPT }, ...normalized]; } -async function runWebSearchTool(input: unknown): Promise { - const args = WebSearchArgsSchema.parse(input); +async function runExaWebSearchTool(args: WebSearchArgs): Promise { const exa = exaClient(); const response = await exa.search(args.query, { type: args.type ?? "auto", @@ -292,6 +295,7 @@ async function runWebSearchTool(input: unknown): Promise { const results = Array.isArray(response?.results) ? response.results : []; return { ok: true, + searchEngine: "exa", query: args.query, requestId: response?.requestId ?? null, results: results.map((result: any, index: number) => ({ @@ -309,6 +313,40 @@ async function runWebSearchTool(input: unknown): Promise { }; } +async function runSearxngWebSearchTool(args: WebSearchArgs): Promise { + const response = await searchSearxng(args.query, { + numResults: args.numResults ?? DEFAULT_WEB_RESULTS, + includeDomains: args.includeDomains, + excludeDomains: args.excludeDomains, + }); + + return { + ok: true, + searchEngine: "searxng", + query: args.query, + requestId: response.requestId, + results: response.results.map((result, index) => ({ + rank: index + 1, + title: result.title, + url: result.url, + publishedDate: result.publishedDate, + author: null, + summary: result.summary, + text: result.text, + highlights: result.summary ? [clipText(result.summary, 280)] : [], + engines: result.engines, + })), + }; +} + +async function runWebSearchTool(input: unknown): Promise { + const args = WebSearchArgsSchema.parse(input); + if (env.CHAT_WEB_SEARCH_ENGINE === "searxng") { + return runSearxngWebSearchTool(args); + } + return runExaWebSearchTool(args); +} + function assertSafeFetchUrl(urlRaw: string) { const parsed = new URL(urlRaw); if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { diff --git a/server/src/search/searxng.ts b/server/src/search/searxng.ts new file mode 100644 index 0000000..154dc86 --- /dev/null +++ b/server/src/search/searxng.ts @@ -0,0 +1,160 @@ +import { env } from "../env.js"; + +const SEARXNG_TIMEOUT_MS = 12_000; +const DEFAULT_SEARXNG_CATEGORIES = "general"; + +export type SearxngSearchOptions = { + numResults: number; + includeDomains?: string[]; + excludeDomains?: string[]; +}; + +export type SearxngSearchResult = { + title: string | null; + url: string | null; + publishedDate: string | null; + summary: string | null; + text: string | null; + engines: string[]; +}; + +export type SearxngSearchResponse = { + query: string; + requestId: null; + results: SearxngSearchResult[]; +}; + +function clipText(input: string, maxCharacters: number) { + return input.length <= maxCharacters ? input : `${input.slice(0, maxCharacters)}...`; +} + +function compactWhitespace(input: string) { + return input.replace(/\r/g, "").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").replace(/\s+/g, " ").trim(); +} + +function requireSearxngBaseUrl() { + if (!env.SEARXNG_BASE_URL) { + throw new Error("SEARXNG_BASE_URL not set"); + } + return env.SEARXNG_BASE_URL.endsWith("/") ? env.SEARXNG_BASE_URL : `${env.SEARXNG_BASE_URL}/`; +} + +function normalizeDomain(input: string) { + const trimmed = input.trim().toLowerCase(); + if (!trimmed) return null; + + try { + const parsed = new URL(trimmed.includes("://") ? trimmed : `https://${trimmed}`); + return parsed.hostname.replace(/^www\./, ""); + } catch { + return trimmed.split(/[/?#]/, 1)[0]?.replace(/^www\./, "") || null; + } +} + +function normalizeDomains(input: string[] | undefined) { + return Array.from(new Set((input ?? []).map(normalizeDomain).filter((domain): domain is string => Boolean(domain)))); +} + +function hostnameMatchesDomain(urlRaw: string | null, domain: string) { + if (!urlRaw) return false; + try { + const hostname = new URL(urlRaw).hostname.toLowerCase().replace(/^www\./, ""); + return hostname === domain || hostname.endsWith(`.${domain}`); + } catch { + return false; + } +} + +function filterResultsByDomains(results: SearxngSearchResult[], options: SearxngSearchOptions) { + const includeDomains = normalizeDomains(options.includeDomains); + const excludeDomains = normalizeDomains(options.excludeDomains); + return results.filter((result) => { + if (includeDomains.length && !includeDomains.some((domain) => hostnameMatchesDomain(result.url, domain))) return false; + if (excludeDomains.some((domain) => hostnameMatchesDomain(result.url, domain))) return false; + return true; + }); +} + +function buildSearxngQuery(query: string, options: SearxngSearchOptions) { + const includeDomains = normalizeDomains(options.includeDomains); + const excludeDomains = normalizeDomains(options.excludeDomains); + const includeClause = + includeDomains.length === 0 + ? "" + : includeDomains.length === 1 + ? `site:${includeDomains[0]}` + : `(${includeDomains.map((domain) => `site:${domain}`).join(" OR ")})`; + const excludeClause = excludeDomains.map((domain) => `-site:${domain}`).join(" "); + return [query, includeClause, excludeClause].filter(Boolean).join(" "); +} + +function buildSearchUrl(query: string, options: SearxngSearchOptions) { + const url = new URL("search", requireSearxngBaseUrl()); + url.searchParams.set("q", buildSearxngQuery(query, options)); + url.searchParams.set("categories", DEFAULT_SEARXNG_CATEGORIES); + url.searchParams.set("language", "auto"); + url.searchParams.set("safesearch", "1"); + url.searchParams.set("format", "json"); + return url; +} + +async function fetchSearxng(url: URL, accept: string) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), SEARXNG_TIMEOUT_MS); + + try { + return await fetch(url, { + redirect: "follow", + signal: controller.signal, + headers: { + "User-Agent": "SybilBot/1.0 (+https://sybil.local)", + Accept: accept, + }, + }); + } finally { + clearTimeout(timeout); + } +} + +function stringOrNull(value: unknown) { + if (typeof value !== "string") return null; + const normalized = compactWhitespace(value); + return normalized || null; +} + +function stringArray(value: unknown) { + if (!Array.isArray(value)) return []; + return value.filter((item): item is string => typeof item === "string").map(compactWhitespace).filter(Boolean); +} + +function mapJsonResult(result: any): SearxngSearchResult { + const summary = stringOrNull(result?.content) ?? stringOrNull(result?.snippet); + const text = summary ? clipText(summary, 700) : null; + return { + title: stringOrNull(result?.title), + url: stringOrNull(result?.url), + publishedDate: stringOrNull(result?.publishedDate) ?? stringOrNull(result?.published_date), + summary: summary ? clipText(summary, 1_400) : null, + text, + engines: stringArray(result?.engines ?? (typeof result?.engine === "string" ? [result.engine] : [])), + }; +} + +export async function searchSearxng(query: string, options: SearxngSearchOptions): Promise { + const url = buildSearchUrl(query, options); + const response = await fetchSearxng(url, "application/json"); + if (!response.ok) { + await response.arrayBuffer(); + throw new Error(`SearXNG JSON search failed with status ${response.status}. Verify search.formats includes json.`); + } + + const contentType = response.headers.get("content-type")?.toLowerCase() ?? ""; + if (!contentType.includes("application/json")) { + await response.arrayBuffer(); + throw new Error(`SearXNG JSON search returned ${contentType || "unknown content type"}.`); + } + + const data: any = await response.json(); + const results = Array.isArray(data?.results) ? data.results.map(mapJsonResult) : []; + return { query, requestId: null, results: filterResultsByDomains(results, options).slice(0, options.numResults) }; +}