Add indexed history search across iOS and web

server/prisma/migrations/20261007000000_add_workspace_full_text_search/migration.sql

@@ -0,0 +1,70 @@
+-- External-content indexes avoid storing a second copy of conversation text.
+-- Keep these virtual tables and triggers in SQL: Prisma does not model FTS5.
+CREATE VIRTUAL TABLE "ChatSearch" USING fts5(
+  title, content='Chat', content_rowid='rowid',
+  tokenize='unicode61 remove_diacritics 2', prefix='1 2 3'
+);
+
+-- Hidden system prompts and tool payloads are not searchable conversation text.
+CREATE VIEW "MessageSearchContent" AS
+  SELECT rowid, content FROM "Message" WHERE role IN ('user', 'assistant');
+CREATE VIRTUAL TABLE "MessageSearch" USING fts5(
+  content, content='MessageSearchContent', content_rowid='rowid',
+  tokenize='unicode61 remove_diacritics 2', prefix='1 2 3'
+);
+
+CREATE VIRTUAL TABLE "SavedSearchSearch" USING fts5(
+  title, query, answerText, content='Search', content_rowid='rowid',
+  tokenize='unicode61 remove_diacritics 2', prefix='1 2 3'
+);
+
+CREATE TRIGGER "ChatSearch_insert" AFTER INSERT ON "Chat" BEGIN
+  INSERT INTO "ChatSearch" (rowid, title) VALUES (new.rowid, new.title);
+END;
+CREATE TRIGGER "ChatSearch_delete" AFTER DELETE ON "Chat" BEGIN
+  INSERT INTO "ChatSearch" ("ChatSearch", rowid, title) VALUES ('delete', old.rowid, old.title);
+END;
+CREATE TRIGGER "ChatSearch_update" AFTER UPDATE OF title ON "Chat"
+WHEN old.title IS NOT new.title BEGIN
+  INSERT INTO "ChatSearch" ("ChatSearch", rowid, title) VALUES ('delete', old.rowid, old.title);
+  INSERT INTO "ChatSearch" (rowid, title) VALUES (new.rowid, new.title);
+END;
+
+CREATE TRIGGER "MessageSearch_insert" AFTER INSERT ON "Message"
+WHEN new.role IN ('user', 'assistant') BEGIN
+  INSERT INTO "MessageSearch" (rowid, content) VALUES (new.rowid, new.content);
+END;
+CREATE TRIGGER "MessageSearch_delete" AFTER DELETE ON "Message"
+WHEN old.role IN ('user', 'assistant') BEGIN
+  INSERT INTO "MessageSearch" ("MessageSearch", rowid, content) VALUES ('delete', old.rowid, old.content);
+END;
+CREATE TRIGGER "MessageSearch_update" AFTER UPDATE OF content, role ON "Message"
+WHEN old.content IS NOT new.content OR old.role IS NOT new.role BEGIN
+  INSERT INTO "MessageSearch" ("MessageSearch", rowid, content)
+    SELECT 'delete', old.rowid, old.content WHERE old.role IN ('user', 'assistant');
+  INSERT INTO "MessageSearch" (rowid, content)
+    SELECT new.rowid, new.content WHERE new.role IN ('user', 'assistant');
+END;
+
+CREATE TRIGGER "SavedSearchSearch_insert" AFTER INSERT ON "Search" BEGIN
+  INSERT INTO "SavedSearchSearch" (rowid, title, query, answerText)
+    VALUES (new.rowid, new.title, new.query, new.answerText);
+END;
+CREATE TRIGGER "SavedSearchSearch_delete" AFTER DELETE ON "Search" BEGIN
+  INSERT INTO "SavedSearchSearch" ("SavedSearchSearch", rowid, title, query, answerText)
+    VALUES ('delete', old.rowid, old.title, old.query, old.answerText);
+END;
+CREATE TRIGGER "SavedSearchSearch_update" AFTER UPDATE OF title, query, answerText ON "Search"
+WHEN old.title IS NOT new.title OR old.query IS NOT new.query OR old.answerText IS NOT new.answerText BEGIN
+  INSERT INTO "SavedSearchSearch" ("SavedSearchSearch", rowid, title, query, answerText)
+    VALUES ('delete', old.rowid, old.title, old.query, old.answerText);
+  INSERT INTO "SavedSearchSearch" (rowid, title, query, answerText)
+    VALUES (new.rowid, new.title, new.query, new.answerText);
+END;
+
+-- Index existing history during migration, then maintain it transactionally.
+INSERT INTO "ChatSearch" ("ChatSearch") VALUES ('rebuild');
+INSERT INTO "MessageSearch" ("MessageSearch") VALUES ('rebuild');
+INSERT INTO "SavedSearchSearch" ("SavedSearchSearch") VALUES ('rebuild');
+
+CREATE INDEX "Chat_updatedAt_idx" ON "Chat"("updatedAt");

server/prisma/schema.prisma

@@ -76,6 +76,7 @@ model Chat {
 
   @@index([userId])
   @@index([parentChatId])
+  @@index([updatedAt])
 }
 
 model Message {

server/src/routes.ts

@@ -16,6 +16,7 @@ import { openaiClient } from "./llm/providers.js";
 import { serializeProviderFields, toPrismaProvider } from "./llm/provider-ids.js";
 import { exaClient } from "./search/exa.js";
 import { isFreshSearchCacheHit, normalizeSearchQuery } from "./search-cache.js";
+import { searchWorkspace } from "./workspace-search.js";
 import type { ChatAttachment } from "./llm/types.js";
 
 const ProviderSchema = z.enum(["openai", "anthropic", "xai", "gemini", "hermes-agent"]);
@@ -927,6 +928,37 @@ export async function registerRoutes(app: FastifyInstance) {
 
   app.get("/v1/workspace-items", async (req) => {
     requireAdmin(req);
+    const parsed = z.object({ q: z.string().trim().max(200).optional() }).safeParse(req.query);
+    if (!parsed.success) throw app.httpErrors.badRequest("q must be a string of at most 200 characters");
+    if (parsed.data.q) {
+      const { hits, hasMore } = await searchWorkspace(parsed.data.q);
+      const [chats, searches] = await Promise.all([
+        prisma.chat.findMany({
+          where: { id: { in: hits.filter((hit) => hit.type === "chat").map((hit) => hit.id) } },
+          select: {
+            ...chatSummarySelect,
+            parentChat: { select: { projectItems: starredProjectItemsSelect } },
+          },
+        }),
+        prisma.search.findMany({
+          where: { id: { in: hits.filter((hit) => hit.type === "search").map((hit) => hit.id) } },
+          select: searchSummarySelect,
+        }),
+      ]);
+      const summaries = new Map<string, Record<string, unknown>>([
+        ...chats.map(({ parentChat, ...chat }) => [
+          `chat:${chat.id}`,
+          // Search omits unmatched roots, but sidebar fork stars belong to the root.
+          { ...serializeChatLike(chat), ...serializeStarFields(parentChat ?? chat) },
+        ] as const),
+        ...searches.map((search) => [`search:${search.id}`, serializeSearchLike(search)] as const),
+      ]);
+      const items = hits.flatMap((hit) => {
+        const summary = summaries.get(`${hit.type}:${hit.id}`);
+        return summary ? [{ ...summary, ...hit }] : [];
+      });
+      return { items, hasMore };
+    }
     return { items: await listWorkspaceItems() };
   });
 

server/src/workspace-search.ts

@@ -0,0 +1,74 @@
+import { Prisma } from "@prisma/client";
+import { prisma } from "./db.js";
+
+export const WORKSPACE_SEARCH_LIMIT = 100;
+
+// Treat input as literal Unicode words, never FTS operators/column expressions.
+// Prefix matching supports type-ahead, including short terms such as "AI".
+export function workspaceSearchExpression(query: string): string | null {
+  const words = query.match(/[\p{L}\p{N}\p{M}\p{Co}]+/gu);
+  return words?.map((word) => `"${word}"*`).join(" AND ") ?? null;
+}
+
+export type WorkspaceSearchHit = {
+  type: "chat" | "search";
+  id: string;
+  searchSnippet: string | null;
+};
+
+export function workspaceSearchSQL(expression: string) {
+  // Batch excerpts after limiting matches. Unary + prevents SQLite from using
+  // the IN clause as repeated rowid lookups (and restarting FTS for every row).
+  // FTS still uses MATCH and the rowid bounds; IN filters that single scan.
+  return Prisma.sql`
+    WITH chat_hits AS (
+      SELECT c.id, NULL AS message_rowid
+      FROM "ChatSearch" JOIN "Chat" c ON c.rowid = "ChatSearch".rowid
+      WHERE "ChatSearch" MATCH ${expression}
+      UNION ALL
+      SELECT m.chatId AS id, MIN(m.rowid) AS message_rowid
+      FROM "MessageSearch" JOIN "Message" m ON m.rowid = "MessageSearch".rowid
+      WHERE "MessageSearch" MATCH ${expression}
+      GROUP BY m.chatId
+    ), matches AS (
+      SELECT 'chat' AS type, c.id, c.updatedAt, MIN(h.message_rowid) AS message_rowid,
+        NULL AS search_rowid
+      FROM chat_hits h JOIN "Chat" c ON c.id = h.id
+      GROUP BY c.id
+      UNION ALL
+      SELECT 'search' AS type, s.id, s.updatedAt, NULL AS message_rowid, s.rowid AS search_rowid
+      FROM "SavedSearchSearch" JOIN "Search" s ON s.rowid = "SavedSearchSearch".rowid
+      WHERE "SavedSearchSearch" MATCH ${expression}
+    ), limited AS MATERIALIZED (
+      SELECT * FROM matches ORDER BY updatedAt DESC, type, id LIMIT ${WORKSPACE_SEARCH_LIMIT + 1}
+    ), message_excerpts AS MATERIALIZED (
+      SELECT rowid, substr(snippet("MessageSearch", 0, '', '', ' … ', 24), 1, 480) AS snippet
+      FROM "MessageSearch"
+      WHERE "MessageSearch" MATCH ${expression}
+        AND rowid >= (SELECT MIN(message_rowid) FROM limited)
+        AND rowid <= (SELECT MAX(message_rowid) FROM limited)
+        AND +rowid IN (SELECT message_rowid FROM limited)
+    ), search_excerpts AS MATERIALIZED (
+      SELECT rowid, substr(snippet("SavedSearchSearch", -1, '', '', ' … ', 24), 1, 480) AS snippet
+      FROM "SavedSearchSearch"
+      WHERE "SavedSearchSearch" MATCH ${expression}
+        AND rowid >= (SELECT MIN(search_rowid) FROM limited)
+        AND rowid <= (SELECT MAX(search_rowid) FROM limited)
+        AND +rowid IN (SELECT search_rowid FROM limited)
+    )
+    SELECT type, id, COALESCE(m.snippet, s.snippet) AS searchSnippet
+    FROM limited
+    LEFT JOIN message_excerpts m ON m.rowid = limited.message_rowid
+    LEFT JOIN search_excerpts s ON s.rowid = limited.search_rowid
+    ORDER BY updatedAt DESC, type, id
+  `;
+}
+
+export async function searchWorkspace(query: string) {
+  const expression = workspaceSearchExpression(query);
+  if (!expression) return { hits: [], hasMore: false };
+  // Match via the inverted indexes first. Hydrate summaries and generate excerpts
+  // only for the bounded result set, never fetch full transcripts into JS.
+  const hits = await prisma.$queryRaw<WorkspaceSearchHit[]>(workspaceSearchSQL(expression));
+  return { hits: hits.slice(0, WORKSPACE_SEARCH_LIMIT), hasMore: hits.length > WORKSPACE_SEARCH_LIMIT };
+}

server/tests/chat-forks.test.ts

@@ -1,6 +1,6 @@
 import assert from "node:assert/strict";
 import { execFileSync } from "node:child_process";
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { dirname, join, resolve } from "node:path";
 import test from "node:test";
@@ -11,6 +11,7 @@ import sensible from "@fastify/sensible";
 const serverRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const databaseDir = mkdtempSync(join(tmpdir(), "sybil-chat-forks-"));
 process.env.DATABASE_URL = `file:${join(databaseDir, "test.db")}`;
+writeFileSync(join(databaseDir, "test.db"), "");
 process.env.OPENAI_API_KEY = "";
 delete process.env.ADMIN_TOKEN;
 

server/tests/workspace-search.test.ts

@@ -0,0 +1,188 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { cpSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+import Fastify from "fastify";
+import sensible from "@fastify/sensible";
+import { Prisma } from "@prisma/client";
+
+const serverRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const databaseDir = mkdtempSync(join(tmpdir(), "sybil-workspace-search-"));
+const migration = "20261007000000_add_workspace_full_text_search";
+process.env.DATABASE_URL = `file:${join(databaseDir, "test.db")}`;
+process.env.OPENAI_API_KEY = "";
+delete process.env.ADMIN_TOKEN;
+writeFileSync(join(databaseDir, "test.db"), "");
+cpSync(join(serverRoot, "prisma/schema.prisma"), join(databaseDir, "schema.prisma"));
+for (const name of readdirSync(join(serverRoot, "prisma/migrations"))) {
+  if (name !== migration) cpSync(join(serverRoot, "prisma/migrations", name), join(databaseDir, "migrations", name), { recursive: true });
+}
+function migrate() {
+  execFileSync(process.execPath, [join(serverRoot, "node_modules/prisma/build/index.js"), "migrate", "deploy", "--schema", join(databaseDir, "schema.prisma")], {
+    cwd: serverRoot, env: process.env, stdio: "pipe",
+  });
+}
+migrate();
+const { prisma } = await import("../src/db.js");
+const oldChat = await prisma.chat.create({ data: {
+  title: "Old constellation notes",
+  messages: { create: [
+    { role: "user", content: "We photographed a nebula before the migration." },
+    { role: "system", content: "secretmigrationprompt" },
+  ] },
+} });
+const oldSearch = await prisma.search.create({ data: { query: "constellation map", answerText: "The nebula is visible." } });
+cpSync(join(serverRoot, "prisma/migrations", migration), join(databaseDir, "migrations", migration), { recursive: true });
+migrate();
+
+const { registerRoutes } = await import("../src/routes.js");
+const { workspaceSearchExpression, workspaceSearchSQL } = await import("../src/workspace-search.js");
+const app = Fastify({ logger: false });
+await app.register(sensible);
+await registerRoutes(app);
+await app.ready();
+
+async function search(query: string) {
+  const response = await app.inject({ method: "GET", url: `/v1/workspace-items?q=${encodeURIComponent(query)}` });
+  assert.equal(response.statusCode, 200, response.body);
+  return response.json();
+}
+
+test.afterEach(async () => {
+  await prisma.chat.deleteMany();
+  await prisma.search.deleteMany();
+});
+test.after(async () => {
+  await app.close();
+  await prisma.$disconnect();
+  rmSync(databaseDir, { recursive: true, force: true });
+});
+
+test("migration indexes existing titles, messages, queries, and answers", async () => {
+  assert.deepEqual(new Set((await search("nebula")).items.map((item: any) => item.id)), new Set([oldChat.id, oldSearch.id]));
+  assert.equal((await search("constellation")).items.length, 2);
+  assert.equal((await search("secretmigrationprompt")).items.length, 0);
+});
+
+test("matches full history, deduplicates messages, and excludes hidden content", async () => {
+  const chat = await prisma.chat.create({ data: {
+    title: "Travel notes", updatedAt: new Date("2000-01-01"),
+    messages: { create: [
+      { role: "user", content: "Let's discuss café planning in Zürich" },
+      { role: "assistant", content: "CAFÉ PLANNING includes booking a table" },
+      { role: "system", content: "hiddensystemvalue" },
+      { role: "tool", content: "hiddentoolvalue" },
+    ] },
+  } });
+  await prisma.chat.createMany({ data: Array.from({ length: 105 }, (_, i) => ({ title: `Recent ${i}` })) });
+  const recent = (await app.inject({ method: "GET", url: "/v1/workspace-items" })).json();
+  assert.equal(recent.items.some((item: any) => item.id === chat.id), false);
+  const result = await search("CAFE plan");
+  assert.equal(result.hasMore, false);
+  assert.deepEqual(result.items.map((item: any) => item.id), [chat.id]);
+  assert.match(result.items[0].searchSnippet, /café planning/i);
+  assert.equal(result.items[0].messages, undefined);
+  assert.equal((await search("zuri")).items.length, 1);
+  assert.equal((await search("hiddensystemvalue")).items.length, 0);
+  assert.equal((await search("hiddentoolvalue")).items.length, 0);
+});
+
+test("indexes edits, role changes, forks, rollbacks, and cascading deletes", async () => {
+  const chat = await prisma.chat.create({ data: {
+    title: "originaltitle", messages: { create: { role: "assistant", content: "originalbody" } },
+  }, include: { messages: true } });
+  const messageID = chat.messages[0].id;
+  await prisma.chat.update({ where: { id: chat.id }, data: { title: "replacementtitle" } });
+  await prisma.message.update({ where: { id: messageID }, data: { content: "replacementbody" } });
+  assert.equal((await search("original")).items.length, 0);
+  assert.equal((await search("replacementbody")).items.length, 1);
+  await prisma.message.update({ where: { id: messageID }, data: { role: "tool" } });
+  assert.equal((await search("replacementbody")).items.length, 0);
+  await prisma.message.update({ where: { id: messageID }, data: { role: "user" } });
+  assert.equal((await search("replacementbody")).items.length, 1);
+  await assert.rejects(prisma.$transaction(async (tx) => {
+    await tx.message.update({ where: { id: messageID }, data: { content: "rolledbackvalue" } });
+    throw new Error("rollback");
+  }));
+  assert.equal((await search("rolledbackvalue")).items.length, 0);
+  assert.equal((await search("replacementbody")).items.length, 1);
+  const fork = await app.inject({ method: "POST", url: `/v1/chats/${chat.id}/fork`, payload: {} });
+  assert.equal(fork.statusCode, 200, fork.body);
+  assert.equal((await search("replacementbody")).items.length, 2);
+  await prisma.message.delete({ where: { id: messageID } });
+  const onlyFork = (await search("replacementbody")).items;
+  assert.deepEqual(onlyFork.map((item: any) => item.id), [fork.json().chat.id]);
+  const starResponse = await app.inject({ method: "PATCH", url: `/v1/chats/${chat.id}/star`, payload: { starred: true } });
+  assert.equal(starResponse.statusCode, 200, starResponse.body);
+  const starredFork = (await search("replacementbody")).items[0];
+  assert.equal(starredFork.starred, true);
+  assert.ok(starredFork.starredAt);
+  assert.equal(starredFork.parentChat, undefined);
+  await prisma.chat.delete({ where: { id: chat.id } });
+  assert.equal((await search("replacement")).items.length, 0);
+  for (const table of ["ChatSearch", "MessageSearch", "SavedSearchSearch"]) {
+    await prisma.$executeRawUnsafe(`INSERT INTO "${table}" ("${table}", rank) VALUES ('integrity-check', 1)`);
+  }
+});
+
+test("saved searches match query and answer, and update/delete their index", async () => {
+  const saved = await prisma.search.create({ data: { title: "Bookmarks", query: "AI design", answerText: "Use a nebula illustration" } });
+  assert.equal((await search("ai des")).items[0].type, "search");
+  assert.match((await search("nebula")).items[0].searchSnippet, /nebula/);
+  await prisma.search.update({ where: { id: saved.id }, data: { title: "New title", query: "newquery", answerText: "newanswer" } });
+  assert.equal((await search("nebula")).items.length, 0);
+  assert.equal((await search("design")).items.length, 0);
+  assert.equal((await search("Bookmarks")).items.length, 0);
+  assert.equal((await search("newanswer")).items.length, 1);
+  await prisma.search.delete({ where: { id: saved.id } });
+  assert.equal((await search("new")).items.length, 0);
+});
+
+test("literal queries, validation, blank queries, and bounded snippets", async () => {
+  await prisma.chat.create({ data: { title: "AI OR safety", messages: { create: { role: "user", content: "longword".repeat(1000) } } } });
+  assert.equal((await search('"AI" OR safety*')).items.length, 1);
+  assert.equal((await search("AI OR other")).items.length, 0);
+  assert.equal((await search("AI")).items.length, 1);
+  assert.equal((await search("*** : () --")).items.length, 0);
+  assert.equal((await search("' UNION SELECT * FROM Chat; --")).items.length, 0);
+  assert.equal((await search("longword")).items[0].searchSnippet.length, 480);
+  const recent = (await app.inject({ method: "GET", url: "/v1/workspace-items" })).json();
+  assert.deepEqual(await search("  "), recent);
+  for (const query of ["q=" + "x".repeat(201), "q=one&q=two"]) {
+    const response = await app.inject({ method: "GET", url: `/v1/workspace-items?${query}` });
+    assert.equal(response.statusCode, 400, response.body);
+  }
+});
+
+test("caps combined results at 100 with deterministic newest-first ordering", async () => {
+  const date = new Date("2026-01-01");
+  await prisma.chat.createMany({ data: Array.from({ length: 100 }, (_, i) => ({ id: `chat-${String(i).padStart(3, "0")}`, title: "commonword", updatedAt: date })) });
+  await prisma.search.create({ data: { id: "saved", query: "commonword", updatedAt: new Date("2026-02-01") } });
+  const result = await search("common");
+  assert.equal(result.items.length, 100);
+  assert.equal(result.hasMore, true);
+  assert.equal(result.items[0].id, "saved");
+  assert.equal(result.items[1].id, "chat-000");
+  assert.equal(result.items.at(-1).id, "chat-098");
+});
+
+test("query plan uses all three FTS indexes", async () => {
+  const plan = await prisma.$queryRaw<Array<{ detail: string }>>(Prisma.sql`EXPLAIN QUERY PLAN ${workspaceSearchSQL(workspaceSearchExpression("needle")!)}`);
+  for (const table of ["ChatSearch", "MessageSearch", "SavedSearchSearch"]) {
+    assert.ok(plan.some((row) => row.detail.includes(table) && /VIRTUAL TABLE INDEX.*M\d/.test(row.detail)), plan.map((row) => row.detail).join("\n"));
+  }
+  assert.equal(plan.some((row) => row.detail.includes("CORRELATED SCALAR SUBQUERY")), false, "Excerpts must not restart FTS for each result");
+});
+
+test("vacuum preserves external-content index row identities", async () => {
+  const removed = await prisma.chat.create({ data: { title: "removed" } });
+  const kept = await prisma.chat.create({ data: { title: "kepttitle", messages: { create: { role: "user", content: "keptbody" } } } });
+  await prisma.chat.delete({ where: { id: removed.id } });
+  await prisma.$executeRaw`VACUUM`;
+  assert.equal((await search("kepttitle")).items[0].id, kept.id);
+  assert.equal((await search("kepttitle")).items[0].searchSnippet, null);
+  assert.equal((await search("keptbody")).items[0].searchSnippet, "keptbody");
+});