Add indexed history search across iOS and web

server/tests/workspace-search.test.ts

@@ -0,0 +1,188 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { cpSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+import Fastify from "fastify";
+import sensible from "@fastify/sensible";
+import { Prisma } from "@prisma/client";
+
+const serverRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const databaseDir = mkdtempSync(join(tmpdir(), "sybil-workspace-search-"));
+const migration = "20261007000000_add_workspace_full_text_search";
+process.env.DATABASE_URL = `file:${join(databaseDir, "test.db")}`;
+process.env.OPENAI_API_KEY = "";
+delete process.env.ADMIN_TOKEN;
+writeFileSync(join(databaseDir, "test.db"), "");
+cpSync(join(serverRoot, "prisma/schema.prisma"), join(databaseDir, "schema.prisma"));
+for (const name of readdirSync(join(serverRoot, "prisma/migrations"))) {
+  if (name !== migration) cpSync(join(serverRoot, "prisma/migrations", name), join(databaseDir, "migrations", name), { recursive: true });
+}
+function migrate() {
+  execFileSync(process.execPath, [join(serverRoot, "node_modules/prisma/build/index.js"), "migrate", "deploy", "--schema", join(databaseDir, "schema.prisma")], {
+    cwd: serverRoot, env: process.env, stdio: "pipe",
+  });
+}
+migrate();
+const { prisma } = await import("../src/db.js");
+const oldChat = await prisma.chat.create({ data: {
+  title: "Old constellation notes",
+  messages: { create: [
+    { role: "user", content: "We photographed a nebula before the migration." },
+    { role: "system", content: "secretmigrationprompt" },
+  ] },
+} });
+const oldSearch = await prisma.search.create({ data: { query: "constellation map", answerText: "The nebula is visible." } });
+cpSync(join(serverRoot, "prisma/migrations", migration), join(databaseDir, "migrations", migration), { recursive: true });
+migrate();
+
+const { registerRoutes } = await import("../src/routes.js");
+const { workspaceSearchExpression, workspaceSearchSQL } = await import("../src/workspace-search.js");
+const app = Fastify({ logger: false });
+await app.register(sensible);
+await registerRoutes(app);
+await app.ready();
+
+async function search(query: string) {
+  const response = await app.inject({ method: "GET", url: `/v1/workspace-items?q=${encodeURIComponent(query)}` });
+  assert.equal(response.statusCode, 200, response.body);
+  return response.json();
+}
+
+test.afterEach(async () => {
+  await prisma.chat.deleteMany();
+  await prisma.search.deleteMany();
+});
+test.after(async () => {
+  await app.close();
+  await prisma.$disconnect();
+  rmSync(databaseDir, { recursive: true, force: true });
+});
+
+test("migration indexes existing titles, messages, queries, and answers", async () => {
+  assert.deepEqual(new Set((await search("nebula")).items.map((item: any) => item.id)), new Set([oldChat.id, oldSearch.id]));
+  assert.equal((await search("constellation")).items.length, 2);
+  assert.equal((await search("secretmigrationprompt")).items.length, 0);
+});
+
+test("matches full history, deduplicates messages, and excludes hidden content", async () => {
+  const chat = await prisma.chat.create({ data: {
+    title: "Travel notes", updatedAt: new Date("2000-01-01"),
+    messages: { create: [
+      { role: "user", content: "Let's discuss café planning in Zürich" },
+      { role: "assistant", content: "CAFÉ PLANNING includes booking a table" },
+      { role: "system", content: "hiddensystemvalue" },
+      { role: "tool", content: "hiddentoolvalue" },
+    ] },
+  } });
+  await prisma.chat.createMany({ data: Array.from({ length: 105 }, (_, i) => ({ title: `Recent ${i}` })) });
+  const recent = (await app.inject({ method: "GET", url: "/v1/workspace-items" })).json();
+  assert.equal(recent.items.some((item: any) => item.id === chat.id), false);
+  const result = await search("CAFE plan");
+  assert.equal(result.hasMore, false);
+  assert.deepEqual(result.items.map((item: any) => item.id), [chat.id]);
+  assert.match(result.items[0].searchSnippet, /café planning/i);
+  assert.equal(result.items[0].messages, undefined);
+  assert.equal((await search("zuri")).items.length, 1);
+  assert.equal((await search("hiddensystemvalue")).items.length, 0);
+  assert.equal((await search("hiddentoolvalue")).items.length, 0);
+});
+
+test("indexes edits, role changes, forks, rollbacks, and cascading deletes", async () => {
+  const chat = await prisma.chat.create({ data: {
+    title: "originaltitle", messages: { create: { role: "assistant", content: "originalbody" } },
+  }, include: { messages: true } });
+  const messageID = chat.messages[0].id;
+  await prisma.chat.update({ where: { id: chat.id }, data: { title: "replacementtitle" } });
+  await prisma.message.update({ where: { id: messageID }, data: { content: "replacementbody" } });
+  assert.equal((await search("original")).items.length, 0);
+  assert.equal((await search("replacementbody")).items.length, 1);
+  await prisma.message.update({ where: { id: messageID }, data: { role: "tool" } });
+  assert.equal((await search("replacementbody")).items.length, 0);
+  await prisma.message.update({ where: { id: messageID }, data: { role: "user" } });
+  assert.equal((await search("replacementbody")).items.length, 1);
+  await assert.rejects(prisma.$transaction(async (tx) => {
+    await tx.message.update({ where: { id: messageID }, data: { content: "rolledbackvalue" } });
+    throw new Error("rollback");
+  }));
+  assert.equal((await search("rolledbackvalue")).items.length, 0);
+  assert.equal((await search("replacementbody")).items.length, 1);
+  const fork = await app.inject({ method: "POST", url: `/v1/chats/${chat.id}/fork`, payload: {} });
+  assert.equal(fork.statusCode, 200, fork.body);
+  assert.equal((await search("replacementbody")).items.length, 2);
+  await prisma.message.delete({ where: { id: messageID } });
+  const onlyFork = (await search("replacementbody")).items;
+  assert.deepEqual(onlyFork.map((item: any) => item.id), [fork.json().chat.id]);
+  const starResponse = await app.inject({ method: "PATCH", url: `/v1/chats/${chat.id}/star`, payload: { starred: true } });
+  assert.equal(starResponse.statusCode, 200, starResponse.body);
+  const starredFork = (await search("replacementbody")).items[0];
+  assert.equal(starredFork.starred, true);
+  assert.ok(starredFork.starredAt);
+  assert.equal(starredFork.parentChat, undefined);
+  await prisma.chat.delete({ where: { id: chat.id } });
+  assert.equal((await search("replacement")).items.length, 0);
+  for (const table of ["ChatSearch", "MessageSearch", "SavedSearchSearch"]) {
+    await prisma.$executeRawUnsafe(`INSERT INTO "${table}" ("${table}", rank) VALUES ('integrity-check', 1)`);
+  }
+});
+
+test("saved searches match query and answer, and update/delete their index", async () => {
+  const saved = await prisma.search.create({ data: { title: "Bookmarks", query: "AI design", answerText: "Use a nebula illustration" } });
+  assert.equal((await search("ai des")).items[0].type, "search");
+  assert.match((await search("nebula")).items[0].searchSnippet, /nebula/);
+  await prisma.search.update({ where: { id: saved.id }, data: { title: "New title", query: "newquery", answerText: "newanswer" } });
+  assert.equal((await search("nebula")).items.length, 0);
+  assert.equal((await search("design")).items.length, 0);
+  assert.equal((await search("Bookmarks")).items.length, 0);
+  assert.equal((await search("newanswer")).items.length, 1);
+  await prisma.search.delete({ where: { id: saved.id } });
+  assert.equal((await search("new")).items.length, 0);
+});
+
+test("literal queries, validation, blank queries, and bounded snippets", async () => {
+  await prisma.chat.create({ data: { title: "AI OR safety", messages: { create: { role: "user", content: "longword".repeat(1000) } } } });
+  assert.equal((await search('"AI" OR safety*')).items.length, 1);
+  assert.equal((await search("AI OR other")).items.length, 0);
+  assert.equal((await search("AI")).items.length, 1);
+  assert.equal((await search("*** : () --")).items.length, 0);
+  assert.equal((await search("' UNION SELECT * FROM Chat; --")).items.length, 0);
+  assert.equal((await search("longword")).items[0].searchSnippet.length, 480);
+  const recent = (await app.inject({ method: "GET", url: "/v1/workspace-items" })).json();
+  assert.deepEqual(await search("  "), recent);
+  for (const query of ["q=" + "x".repeat(201), "q=one&q=two"]) {
+    const response = await app.inject({ method: "GET", url: `/v1/workspace-items?${query}` });
+    assert.equal(response.statusCode, 400, response.body);
+  }
+});
+
+test("caps combined results at 100 with deterministic newest-first ordering", async () => {
+  const date = new Date("2026-01-01");
+  await prisma.chat.createMany({ data: Array.from({ length: 100 }, (_, i) => ({ id: `chat-${String(i).padStart(3, "0")}`, title: "commonword", updatedAt: date })) });
+  await prisma.search.create({ data: { id: "saved", query: "commonword", updatedAt: new Date("2026-02-01") } });
+  const result = await search("common");
+  assert.equal(result.items.length, 100);
+  assert.equal(result.hasMore, true);
+  assert.equal(result.items[0].id, "saved");
+  assert.equal(result.items[1].id, "chat-000");
+  assert.equal(result.items.at(-1).id, "chat-098");
+});
+
+test("query plan uses all three FTS indexes", async () => {
+  const plan = await prisma.$queryRaw<Array<{ detail: string }>>(Prisma.sql`EXPLAIN QUERY PLAN ${workspaceSearchSQL(workspaceSearchExpression("needle")!)}`);
+  for (const table of ["ChatSearch", "MessageSearch", "SavedSearchSearch"]) {
+    assert.ok(plan.some((row) => row.detail.includes(table) && /VIRTUAL TABLE INDEX.*M\d/.test(row.detail)), plan.map((row) => row.detail).join("\n"));
+  }
+  assert.equal(plan.some((row) => row.detail.includes("CORRELATED SCALAR SUBQUERY")), false, "Excerpts must not restart FTS for each result");
+});
+
+test("vacuum preserves external-content index row identities", async () => {
+  const removed = await prisma.chat.create({ data: { title: "removed" } });
+  const kept = await prisma.chat.create({ data: { title: "kepttitle", messages: { create: { role: "user", content: "keptbody" } } } });
+  await prisma.chat.delete({ where: { id: removed.id } });
+  await prisma.$executeRaw`VACUUM`;
+  assert.equal((await search("kepttitle")).items[0].id, kept.id);
+  assert.equal((await search("kepttitle")).items[0].searchSnippet, null);
+  assert.equal((await search("keptbody")).items[0].searchSnippet, "keptbody");
+});