adds search support with exa

2026-02-13 23:49:55 -08:00
parent 5340c55aa6
commit 393dac37a7
14 changed files with 948 additions and 155 deletions
--- a/server/.gitignore
+++ b/server/.gitignore
@@ -4,6 +4,5 @@ node_modules
 dev.db
 *.db
 *.db-journal
-prisma/migrations
 .DS_Store
 dist
--- a/server/README.md
+++ b/server/README.md
@@ -39,6 +39,7 @@ If `ADMIN_TOKEN` is not set, the server runs in open mode (dev).
 - `OPENAI_API_KEY`
 - `ANTHROPIC_API_KEY`
 - `XAI_API_KEY`
+- `EXA_API_KEY`

 ## API
 - `GET /health`
@@ -49,6 +50,10 @@ If `ADMIN_TOKEN` is not set, the server runs in open mode (dev).
 - `POST /v1/chats/:chatId/messages`
 - `POST /v1/chat-completions`
 - `POST /v1/chat-completions/stream` (SSE)
+- `GET /v1/searches`
+- `POST /v1/searches`
+- `GET /v1/searches/:searchId`
+- `POST /v1/searches/:searchId/run`

 When `chatId` is provided to completion endpoints, you can send full conversation context. The server now stores only new non-assistant messages to avoid duplicate history rows.

--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -16,6 +16,7 @@
        "@fastify/swagger-ui": "^5.2.5",
        "@prisma/client": "^6.6.0",
        "dotenv": "^17.2.3",
+        "exa-js": "^2.4.0",
        "fastify": "^5.7.2",
        "openai": "^6.16.0",
        "pino-pretty": "^13.1.3",
@@ -960,6 +961,15 @@
        "url": "https://opencollective.com/express"
      }
    },
+    "node_modules/cross-fetch": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.1.0.tgz",
+      "integrity": "sha512-uKm5PU+MHTootlWEY+mZ4vvXoCn4fLQxT9dSc1sXVMSFkINTJVN8cAQROpwcKm8bJ/c7rgZVIBWzH5T78sNZZw==",
+      "license": "MIT",
+      "dependencies": {
+        "node-fetch": "^2.7.0"
+      }
+    },
    "node_modules/dateformat": {
      "version": "4.6.3",
      "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",
@@ -1084,6 +1094,61 @@
      "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==",
      "license": "MIT"
    },
+    "node_modules/exa-js": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/exa-js/-/exa-js-2.4.0.tgz",
+      "integrity": "sha512-zOFClWWZnh9wyUN3xiBgbhuT8DsS62uZJY+P9toN4KgxyCRQma7aU89/7UCtrXNwq5kEFAACw4eualXcTKjiAQ==",
+      "license": "MIT",
+      "dependencies": {
+        "cross-fetch": "~4.1.0",
+        "dotenv": "~16.4.7",
+        "openai": "^5.0.1",
+        "zod": "^3.22.0",
+        "zod-to-json-schema": "^3.20.0"
+      }
+    },
+    "node_modules/exa-js/node_modules/dotenv": {
+      "version": "16.4.7",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz",
+      "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==",
+      "license": "BSD-2-Clause",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://dotenvx.com"
+      }
+    },
+    "node_modules/exa-js/node_modules/openai": {
+      "version": "5.23.2",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-5.23.2.tgz",
+      "integrity": "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg==",
+      "license": "Apache-2.0",
+      "bin": {
+        "openai": "bin/cli"
+      },
+      "peerDependencies": {
+        "ws": "^8.18.0",
+        "zod": "^3.23.8"
+      },
+      "peerDependenciesMeta": {
+        "ws": {
+          "optional": true
+        },
+        "zod": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/exa-js/node_modules/zod": {
+      "version": "3.25.76",
+      "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
+      "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
+      "license": "MIT",
+      "funding": {
+        "url": "https://github.com/sponsors/colinhacks"
+      }
+    },
    "node_modules/fast-copy": {
      "version": "4.0.2",
      "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.0.2.tgz",
@@ -1518,6 +1583,26 @@
      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
      "license": "MIT"
    },
+    "node_modules/node-fetch": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+      "license": "MIT",
+      "dependencies": {
+        "whatwg-url": "^5.0.0"
+      },
+      "engines": {
+        "node": "4.x || >=6.0.0"
+      },
+      "peerDependencies": {
+        "encoding": "^0.1.0"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/on-exit-leak-free": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
@@ -1890,6 +1975,12 @@
        "node": ">=0.6"
      }
    },
+    "node_modules/tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
+      "license": "MIT"
+    },
    "node_modules/ts-algebra": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz",
@@ -1960,6 +2051,22 @@
        "node": ">= 0.8"
      }
    },
+    "node_modules/webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
+      "license": "BSD-2-Clause"
+    },
+    "node_modules/whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+      "license": "MIT",
+      "dependencies": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    },
    "node_modules/wrappy": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
@@ -1989,6 +2096,15 @@
      "funding": {
        "url": "https://github.com/sponsors/colinhacks"
      }
+    },
+    "node_modules/zod-to-json-schema": {
+      "version": "3.25.1",
+      "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.1.tgz",
+      "integrity": "sha512-pM/SU9d3YAggzi6MtR4h7ruuQlqKtad8e9S0fmxcMi+ueAK5Korys/aWcV9LIIHTVbj01NdzxcnXSN+O74ZIVA==",
+      "license": "ISC",
+      "peerDependencies": {
+        "zod": "^3.25 || ^4"
+      }
    }
  }
 }
--- a/server/package.json
+++ b/server/package.json
@@ -24,10 +24,11 @@
    "@fastify/swagger-ui": "^5.2.5",
    "@prisma/client": "^6.6.0",
    "dotenv": "^17.2.3",
+    "exa-js": "^2.4.0",
    "fastify": "^5.7.2",
    "openai": "^6.16.0",
-    "prisma": "^6.6.0",
    "pino-pretty": "^13.1.3",
+    "prisma": "^6.6.0",
    "zod": "^4.3.6"
  },
  "devDependencies": {
--- a/server/prisma/migrations/20260128083259_/migration.sql
+++ b/server/prisma/migrations/20260128083259_/migration.sql
@@ -0,0 +1,61 @@
+-- CreateTable
+CREATE TABLE "User" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "handle" TEXT
+);
+
+-- CreateTable
+CREATE TABLE "Chat" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "title" TEXT,
+    "userId" TEXT,
+    CONSTRAINT "Chat_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE SET NULL ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "Message" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "chatId" TEXT NOT NULL,
+    "role" TEXT NOT NULL,
+    "content" TEXT NOT NULL,
+    "name" TEXT,
+    "metadata" JSONB,
+    CONSTRAINT "Message_chatId_fkey" FOREIGN KEY ("chatId") REFERENCES "Chat" ("id") ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "LlmCall" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "chatId" TEXT NOT NULL,
+    "provider" TEXT NOT NULL,
+    "model" TEXT NOT NULL,
+    "request" JSONB NOT NULL,
+    "response" JSONB,
+    "inputTokens" INTEGER,
+    "outputTokens" INTEGER,
+    "totalTokens" INTEGER,
+    "latencyMs" INTEGER,
+    "error" TEXT,
+    CONSTRAINT "LlmCall_chatId_fkey" FOREIGN KEY ("chatId") REFERENCES "Chat" ("id") ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "User_handle_key" ON "User"("handle");
+
+-- CreateIndex
+CREATE INDEX "Chat_userId_idx" ON "Chat"("userId");
+
+-- CreateIndex
+CREATE INDEX "Message_chatId_createdAt_idx" ON "Message"("chatId", "createdAt");
+
+-- CreateIndex
+CREATE INDEX "LlmCall_chatId_createdAt_idx" ON "LlmCall"("chatId", "createdAt");
+
+-- CreateIndex
+CREATE INDEX "LlmCall_provider_model_createdAt_idx" ON "LlmCall"("provider", "model", "createdAt");
--- a/server/prisma/migrations/20260214074245_add_exa_search/migration.sql
+++ b/server/prisma/migrations/20260214074245_add_exa_search/migration.sql
@@ -0,0 +1,43 @@
+-- CreateTable
+CREATE TABLE "Search" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" DATETIME NOT NULL,
+    "title" TEXT,
+    "query" TEXT,
+    "source" TEXT NOT NULL DEFAULT 'exa',
+    "requestId" TEXT,
+    "rawResponse" JSONB,
+    "latencyMs" INTEGER,
+    "error" TEXT,
+    "userId" TEXT,
+    CONSTRAINT "Search_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE SET NULL ON UPDATE CASCADE
+);
+
+-- CreateTable
+CREATE TABLE "SearchResult" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "searchId" TEXT NOT NULL,
+    "rank" INTEGER NOT NULL,
+    "title" TEXT,
+    "url" TEXT NOT NULL,
+    "publishedDate" TEXT,
+    "author" TEXT,
+    "text" TEXT,
+    "highlights" JSONB,
+    "highlightScores" JSONB,
+    "score" REAL,
+    "favicon" TEXT,
+    "image" TEXT,
+    CONSTRAINT "SearchResult_searchId_fkey" FOREIGN KEY ("searchId") REFERENCES "Search" ("id") ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+-- CreateIndex
+CREATE INDEX "Search_updatedAt_idx" ON "Search"("updatedAt");
+
+-- CreateIndex
+CREATE INDEX "Search_userId_idx" ON "Search"("userId");
+
+-- CreateIndex
+CREATE INDEX "SearchResult_searchId_rank_idx" ON "SearchResult"("searchId", "rank");
--- a/server/prisma/migrations/migration_lock.toml
+++ b/server/prisma/migrations/migration_lock.toml
@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (e.g., Git)
+provider = "sqlite"
--- a/server/prisma/schema.prisma
+++ b/server/prisma/schema.prisma
@@ -22,6 +22,10 @@ enum MessageRole {
  tool
 }

+enum SearchSource {
+  exa
+}
+
 model User {
  id        String   @id @default(cuid())
  createdAt DateTime @default(now())
@@ -31,6 +35,7 @@ model User {
  handle String? @unique

  chats Chat[]
+  searches Search[]
 }

 model Chat {
@@ -92,3 +97,50 @@ model LlmCall {
  @@index([chatId, createdAt])
  @@index([provider, model, createdAt])
 }
+
+model Search {
+  id        String       @id @default(cuid())
+  createdAt DateTime     @default(now())
+  updatedAt DateTime     @updatedAt
+
+  title String?
+  query String?
+
+  source SearchSource @default(exa)
+
+  requestId   String?
+  rawResponse Json?
+  latencyMs   Int?
+  error       String?
+
+  user   User?   @relation(fields: [userId], references: [id])
+  userId String?
+
+  results SearchResult[]
+
+  @@index([updatedAt])
+  @@index([userId])
+}
+
+model SearchResult {
+  id        String   @id @default(cuid())
+  createdAt DateTime @default(now())
+
+  search   Search @relation(fields: [searchId], references: [id], onDelete: Cascade)
+  searchId String
+
+  rank Int
+
+  title         String?
+  url           String
+  publishedDate String?
+  author        String?
+  text          String?
+  highlights    Json?
+  highlightScores Json?
+  score         Float?
+  favicon       String?
+  image         String?
+
+  @@index([searchId, rank])
+}
--- a/server/src/env.ts
+++ b/server/src/env.ts
@@ -12,6 +12,7 @@ const EnvSchema = z.object({
  OPENAI_API_KEY: z.string().optional(),
  ANTHROPIC_API_KEY: z.string().optional(),
  XAI_API_KEY: z.string().optional(),
+  EXA_API_KEY: z.string().optional(),
 });

 export type Env = z.infer<typeof EnvSchema>;
--- a/server/src/routes.ts
+++ b/server/src/routes.ts
@@ -1,3 +1,4 @@
+import { performance } from "node:perf_hooks";
 import { z } from "zod";
 import type { FastifyInstance } from "fastify";
 import { prisma } from "./db.js";
@@ -5,6 +6,7 @@ import { requireAdmin } from "./auth.js";
 import { env } from "./env.js";
 import { runMultiplex } from "./llm/multiplexer.js";
 import { runMultiplexStream } from "./llm/streaming.js";
+import { exaClient } from "./search/exa.js";

 type IncomingChatMessage = {
  role: "system" | "user" | "assistant" | "tool";
@@ -73,6 +75,141 @@ export async function registerRoutes(app: FastifyInstance) {
    return { chat };
  });

+  app.get("/v1/searches", async (req) => {
+    requireAdmin(req);
+    const searches = await prisma.search.findMany({
+      orderBy: { updatedAt: "desc" },
+      take: 100,
+      select: { id: true, title: true, query: true, createdAt: true, updatedAt: true },
+    });
+    return { searches };
+  });
+
+  app.post("/v1/searches", async (req) => {
+    requireAdmin(req);
+    const Body = z.object({ title: z.string().optional(), query: z.string().optional() });
+    const body = Body.parse(req.body ?? {});
+    const title = body.title?.trim() || body.query?.trim()?.slice(0, 80);
+    const query = body.query?.trim() || null;
+    const search = await prisma.search.create({
+      data: {
+        title: title || null,
+        query,
+      },
+      select: { id: true, title: true, query: true, createdAt: true, updatedAt: true },
+    });
+    return { search };
+  });
+
+  app.get("/v1/searches/:searchId", async (req) => {
+    requireAdmin(req);
+    const Params = z.object({ searchId: z.string() });
+    const { searchId } = Params.parse(req.params);
+
+    const search = await prisma.search.findUnique({
+      where: { id: searchId },
+      include: { results: { orderBy: { rank: "asc" } } },
+    });
+    if (!search) return app.httpErrors.notFound("search not found");
+    return { search };
+  });
+
+  app.post("/v1/searches/:searchId/run", async (req) => {
+    requireAdmin(req);
+    const Params = z.object({ searchId: z.string() });
+    const Body = z.object({
+      query: z.string().trim().min(1).optional(),
+      title: z.string().trim().min(1).optional(),
+      type: z.enum(["auto", "fast", "deep", "instant"]).optional(),
+      numResults: z.number().int().min(1).max(25).optional(),
+      includeDomains: z.array(z.string().trim().min(1)).max(50).optional(),
+      excludeDomains: z.array(z.string().trim().min(1)).max(50).optional(),
+    });
+
+    const { searchId } = Params.parse(req.params);
+    const body = Body.parse(req.body ?? {});
+
+    const existing = await prisma.search.findUnique({
+      where: { id: searchId },
+      select: { id: true, query: true },
+    });
+    if (!existing) return app.httpErrors.notFound("search not found");
+
+    const query = body.query?.trim() || existing.query?.trim();
+    if (!query) return app.httpErrors.badRequest("query is required");
+
+    const startedAt = performance.now();
+
+    try {
+      const response = await exaClient().searchAndContents(query, {
+        type: body.type ?? "auto",
+        numResults: body.numResults ?? 10,
+        includeDomains: body.includeDomains,
+        excludeDomains: body.excludeDomains,
+        text: { maxCharacters: 1200 },
+        highlights: {
+          query,
+          maxCharacters: 320,
+          numSentences: 2,
+          highlightsPerUrl: 2,
+        },
+        moderation: true,
+        userLocation: "US",
+      } as any);
+
+      const latencyMs = Math.round(performance.now() - startedAt);
+      const normalizedTitle = body.title?.trim() || query.slice(0, 80);
+      const rows = (response.results ?? []).map((result: any, index: number) => ({
+        searchId,
+        rank: index,
+        title: result.title ?? null,
+        url: result.url,
+        publishedDate: result.publishedDate ?? null,
+        author: result.author ?? null,
+        text: result.text ?? null,
+        highlights: Array.isArray(result.highlights) ? (result.highlights as any) : null,
+        highlightScores: Array.isArray(result.highlightScores) ? (result.highlightScores as any) : null,
+        score: typeof result.score === "number" ? result.score : null,
+        favicon: result.favicon ?? null,
+        image: result.image ?? null,
+      }));
+
+      await prisma.$transaction(async (tx) => {
+        await tx.search.update({
+          where: { id: searchId },
+          data: {
+            query,
+            title: normalizedTitle,
+            requestId: response.requestId ?? null,
+            rawResponse: response as any,
+            latencyMs,
+            error: null,
+          },
+        });
+        await tx.searchResult.deleteMany({ where: { searchId } });
+        if (rows.length) {
+          await tx.searchResult.createMany({ data: rows as any });
+        }
+      });
+
+      const search = await prisma.search.findUnique({
+        where: { id: searchId },
+        include: { results: { orderBy: { rank: "asc" } } },
+      });
+      if (!search) return app.httpErrors.notFound("search not found");
+      return { search };
+    } catch (err: any) {
+      await prisma.search.update({
+        where: { id: searchId },
+        data: {
+          latencyMs: Math.round(performance.now() - startedAt),
+          error: err?.message ?? String(err),
+        },
+      });
+      throw err;
+    }
+  });
+
  app.get("/v1/chats/:chatId", async (req) => {
    requireAdmin(req);
    const Params = z.object({ chatId: z.string() });
--- a/server/src/search/exa.ts
+++ b/server/src/search/exa.ts
@@ -0,0 +1,14 @@
+import { Exa } from "exa-js";
+import { env } from "../env.js";
+
+let client: Exa | null = null;
+
+export function exaClient() {
+  if (!env.EXA_API_KEY) {
+    throw new Error("EXA_API_KEY not set");
+  }
+  if (!client) {
+    client = new Exa(env.EXA_API_KEY);
+  }
+  return client;
+}