Sybil-2/server/src/search/brave.ts

import { buildBrowserLikeRequestHeaders } from "../browser-fetch-headers.js";
import { env } from "../env.js";

const BRAVE_WEB_SEARCH_URL = "https://api.search.brave.com/res/v1/web/search";
const BRAVE_SEARCH_TIMEOUT_MS = 12_000;

export type BraveSearchOptions = {
  numResults: number;
  includeDomains?: string[];
  excludeDomains?: string[];
};

export type BraveSearchResult = {
  title: string | null;
  url: string | null;
  publishedDate: string | null;
  author: string | null;
  summary: string | null;
  text: string | null;
  highlights: string[];
};

export type BraveSearchResponse = {
  query: string;
  requestId: string | null;
  results: BraveSearchResult[];
};

function clipText(input: string, maxCharacters: number) {
  return input.length <= maxCharacters ? input : `${input.slice(0, maxCharacters)}...`;
}

function compactWhitespace(input: string) {
  return input.replace(/\r/g, "").replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").replace(/\s+/g, " ").trim();
}

function requireBraveSearchApiKey() {
  if (!env.BRAVE_SEARCH_API_KEY) {
    throw new Error("BRAVE_SEARCH_API_KEY not set");
  }
  return env.BRAVE_SEARCH_API_KEY;
}

function normalizeDomain(input: string) {
  const trimmed = input.trim().toLowerCase();
  if (!trimmed) return null;

  try {
    const parsed = new URL(trimmed.includes("://") ? trimmed : `https://${trimmed}`);
    return parsed.hostname.replace(/^www\./, "");
  } catch {
    return trimmed.split(/[/?#]/, 1)[0]?.replace(/^www\./, "") || null;
  }
}

function normalizeDomains(input: string[] | undefined) {
  return Array.from(new Set((input ?? []).map(normalizeDomain).filter((domain): domain is string => Boolean(domain))));
}

function hostnameMatchesDomain(urlRaw: string | null, domain: string) {
  if (!urlRaw) return false;
  try {
    const hostname = new URL(urlRaw).hostname.toLowerCase().replace(/^www\./, "");
    return hostname === domain || hostname.endsWith(`.${domain}`);
  } catch {
    return false;
  }
}

function filterResultsByDomains(results: BraveSearchResult[], options: BraveSearchOptions) {
  const includeDomains = normalizeDomains(options.includeDomains);
  const excludeDomains = normalizeDomains(options.excludeDomains);
  return results.filter((result) => {
    if (includeDomains.length && !includeDomains.some((domain) => hostnameMatchesDomain(result.url, domain))) return false;
    if (excludeDomains.some((domain) => hostnameMatchesDomain(result.url, domain))) return false;
    return true;
  });
}

function buildBraveQuery(query: string, options: BraveSearchOptions) {
  const includeDomains = normalizeDomains(options.includeDomains);
  const excludeDomains = normalizeDomains(options.excludeDomains);
  const includeClause =
    includeDomains.length === 0
      ? ""
      : includeDomains.length === 1
        ? `site:${includeDomains[0]}`
        : `(${includeDomains.map((domain) => `site:${domain}`).join(" OR ")})`;
  const excludeClause = excludeDomains.map((domain) => `-site:${domain}`).join(" ");
  return [query, includeClause, excludeClause].filter(Boolean).join(" ");
}

function buildSearchUrl(query: string, options: BraveSearchOptions) {
  const url = new URL(BRAVE_WEB_SEARCH_URL);
  url.searchParams.set("q", buildBraveQuery(query, options));
  url.searchParams.set("count", String(options.numResults));
  url.searchParams.set("safesearch", "moderate");
  url.searchParams.set("result_filter", "web");
  url.searchParams.set("text_decorations", "false");
  url.searchParams.set("extra_snippets", "true");
  return url;
}

function stringOrNull(value: unknown) {
  if (typeof value !== "string") return null;
  const normalized = compactWhitespace(value);
  return normalized || null;
}

function stringArray(value: unknown) {
  if (!Array.isArray(value)) return [];
  return value.filter((item): item is string => typeof item === "string").map(compactWhitespace).filter(Boolean);
}

function mapWebResult(result: any): BraveSearchResult {
  const description = stringOrNull(result?.description);
  const extraSnippets = stringArray(result?.extra_snippets);
  const snippets = [description, ...extraSnippets].filter((snippet): snippet is string => Boolean(snippet));
  const combinedText = snippets.join("\n\n");

  return {
    title: stringOrNull(result?.title),
    url: stringOrNull(result?.url),
    publishedDate: stringOrNull(result?.page_age),
    author: stringOrNull(result?.profile?.name) ?? stringOrNull(result?.article?.author),
    summary: description ? clipText(description, 1_400) : null,
    text: combinedText ? clipText(combinedText, 700) : null,
    highlights: snippets.slice(0, 3).map((snippet) => clipText(snippet, 280)),
  };
}

export async function searchBrave(query: string, options: BraveSearchOptions): Promise<BraveSearchResponse> {
  const url = buildSearchUrl(query, options);
  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), BRAVE_SEARCH_TIMEOUT_MS);

  let response: Response;
  try {
    response = await fetch(url, {
      signal: controller.signal,
      headers: {
        ...buildBrowserLikeRequestHeaders("application/json"),
        "X-Subscription-Token": requireBraveSearchApiKey(),
      },
    });
  } finally {
    clearTimeout(timeout);
  }

  if (!response.ok) {
    await response.arrayBuffer();
    throw new Error(`Brave Search API request failed with status ${response.status}.`);
  }

  const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
  if (!contentType.includes("application/json")) {
    await response.arrayBuffer();
    throw new Error(`Brave Search API returned ${contentType || "unknown content type"}.`);
  }

  const data: any = await response.json();
  const results = Array.isArray(data?.web?.results) ? data.web.results.map(mapWebResult) : [];
  return {
    query,
    requestId: response.headers.get("x-request-id"),
    results: filterResultsByDomains(results, options).slice(0, options.numResults),
  };
}