175 lines
10 KiB
TypeScript
175 lines
10 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { execFileSync } from "node:child_process";
|
|
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { dirname, join, resolve } from "node:path";
|
|
import test from "node:test";
|
|
import { fileURLToPath } from "node:url";
|
|
import Fastify from "fastify";
|
|
import sensible from "@fastify/sensible";
|
|
import { getReasoningCatalog, normalizeReasoningEffort } from "../src/llm/reasoning-effort.js";
|
|
|
|
const serverRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
const databaseDir = mkdtempSync(join(tmpdir(), "sybil-reasoning-"));
|
|
process.env.DATABASE_URL = `file:${join(databaseDir, "test.db")}`;
|
|
writeFileSync(join(databaseDir, "test.db"), "");
|
|
process.env.OPENAI_API_KEY = "test-key";
|
|
process.env.XAI_API_KEY = "test-key";
|
|
delete process.env.ADMIN_TOKEN;
|
|
execFileSync(process.execPath, [join(serverRoot, "node_modules/prisma/build/index.js"), "migrate", "deploy"], {
|
|
cwd: serverRoot, env: { ...process.env, PRISMA_HIDE_UPDATE_MESSAGE: "1" }, stdio: "pipe",
|
|
});
|
|
const [{ registerRoutes }, { prisma }, { getProviderChatAdapter }] = await Promise.all([
|
|
import("../src/routes.js"), import("../src/db.js"), import("../src/llm/provider-adapters.js"),
|
|
]);
|
|
const app = Fastify({ logger: false });
|
|
await app.register(sensible);
|
|
await registerRoutes(app);
|
|
await app.ready();
|
|
|
|
const requests: { path: string; body: any }[] = [];
|
|
const originalFetch = globalThis.fetch;
|
|
globalThis.fetch = async (input, init) => {
|
|
const url = new URL(input instanceof Request ? input.url : String(input));
|
|
assert.ok(["api.openai.com", "api.x.ai"].includes(url.hostname), `Unexpected network request: ${url.origin}`);
|
|
const body = JSON.parse(String(init?.body));
|
|
requests.push({ path: url.pathname, body });
|
|
const message = { id: "message", type: "message", role: "assistant", content: [{ type: "output_text", text: "Mock answer." }] };
|
|
const response = { id: "response", status: "completed", output: [message], output_text: "Mock answer." };
|
|
if (!body.stream) return Response.json(url.pathname.endsWith("/responses") ? response : {
|
|
id: "completion", choices: [{ message: { role: "assistant", content: "Mock answer." }, finish_reason: "stop" }],
|
|
});
|
|
const events = url.pathname.endsWith("/responses") ? [
|
|
{ type: "response.output_item.added", item: message, output_index: 0 },
|
|
{ type: "response.output_text.delta", delta: "Mock answer.", output_index: 0 },
|
|
{ type: "response.completed", response },
|
|
] : [{ choices: [{ index: 0, delta: { content: "Mock answer." }, finish_reason: "stop" }] }];
|
|
return new Response(events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("") + "data: [DONE]\n\n", {
|
|
headers: { "Content-Type": "text/event-stream" },
|
|
});
|
|
};
|
|
test.after(async () => {
|
|
globalThis.fetch = originalFetch;
|
|
await app.close();
|
|
await prisma.$disconnect();
|
|
rmSync(databaseDir, { recursive: true, force: true });
|
|
});
|
|
|
|
test("capabilities require an exact verified provider/model/effort combination", () => {
|
|
assert.equal(normalizeReasoningEffort("openai", "gpt-5.2", "xhigh"), "xhigh");
|
|
assert.equal(normalizeReasoningEffort("openai", "gpt-5.2-2025-12-11", "none"), "none");
|
|
assert.equal(normalizeReasoningEffort("openai", "gpt-5", "minimal"), "minimal");
|
|
for (const [provider, model, effort] of [
|
|
["openai", "gpt-5.1", "xhigh"], ["openai", "gpt-6.1-sol", "none"],
|
|
["openai", "gpt-5.2-future", "high"], ["openai", "gpt-5.2-pro", "high"],
|
|
["openai", "ft:gpt-5.2", "high"], ["openai", "gpt-4.1-mini", "high"],
|
|
["openai", "chat-latest", "high"], ["xai", "gpt-5.2", "high"],
|
|
]) assert.equal(normalizeReasoningEffort(provider, model, effort), null);
|
|
const catalog = getReasoningCatalog("openai");
|
|
assert.equal(catalog["gpt-5.2"].defaultEffort, "none");
|
|
assert.equal(catalog["gpt-5.5"].defaultEffort, "medium");
|
|
assert.deepEqual(getReasoningCatalog("anthropic"), {});
|
|
catalog["gpt-5.2"].efforts.push("max");
|
|
assert.equal(normalizeReasoningEffort("openai", "gpt-5.2", "max"), null);
|
|
});
|
|
|
|
test("real SDK serialization uses the right effort field for every OpenAI protocol and stream mode", async () => {
|
|
for (const enabledTools of [[], ["web_search"]]) {
|
|
for (const streaming of [false, true]) {
|
|
for (const reasoningEffort of ["high", "none", null] as const) {
|
|
const adapter = getProviderChatAdapter("openai");
|
|
const params = { model: "gpt-5.2", messages: [{ role: "user" as const, content: "Hello" }], enabledTools, reasoningEffort, temperature: 0.2, maxTokens: 1000 };
|
|
if (streaming) {
|
|
const events = [];
|
|
for await (const event of adapter.stream(params)) events.push(event);
|
|
assert.equal(events.at(-1)?.type, "done");
|
|
} else assert.equal((await adapter.complete(params)).text, "Mock answer.");
|
|
const { path, body } = requests.at(-1)!;
|
|
assert.equal(path, enabledTools.length ? "/v1/responses" : "/v1/chat/completions");
|
|
if (enabledTools.length) {
|
|
assert.deepEqual(body.reasoning, reasoningEffort ? { effort: reasoningEffort } : undefined);
|
|
assert.equal(body.reasoning_effort, undefined);
|
|
assert.equal(body.max_output_tokens, 1000);
|
|
} else {
|
|
assert.equal(body.reasoning_effort, reasoningEffort ?? undefined);
|
|
assert.equal(body.reasoning, undefined);
|
|
assert.equal(body.max_completion_tokens, 1000);
|
|
assert.equal(body.max_tokens, undefined);
|
|
}
|
|
assert.equal(body.temperature, reasoningEffort === "high" ? undefined : 0.2);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
test("unsupported models, unsupported efforts and other providers never receive an effort field", async () => {
|
|
for (const provider of ["openai", "xai"] as const) {
|
|
for (const model of ["gpt-4.1-mini", "chat-latest", "gpt-5.1", "gpt-5.2-unknown"]) {
|
|
for (const enabledTools of [[], ["web_search"]]) {
|
|
const adapter = getProviderChatAdapter(provider);
|
|
await adapter.complete({ model, messages: [{ role: "user", content: "Hi" }], enabledTools, reasoningEffort: "xhigh" });
|
|
const { body } = requests.at(-1)!;
|
|
assert.equal("reasoning" in body, false);
|
|
assert.equal("reasoning_effort" in body, false);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
test("chat settings persist, survive all summary/detail reads and forks, and reset on a model change", async () => {
|
|
const created = await app.inject({ method: "POST", url: "/v1/chats", payload: { provider: "openai", model: "gpt-5.2", reasoningEffort: "high" } });
|
|
assert.equal(created.statusCode, 200, created.body);
|
|
const chat = created.json().chat;
|
|
assert.equal(chat.reasoningEffort, "high");
|
|
for (const url of [`/v1/chats/${chat.id}`, "/v1/chats", "/v1/chats?limit=10", "/v1/workspace-items"]) {
|
|
const body = (await app.inject({ method: "GET", url })).json();
|
|
const stored = body.chat ?? (body.chats ?? body.items).find((item: any) => item.id === chat.id);
|
|
assert.equal(stored.reasoningEffort, "high", url);
|
|
}
|
|
const fork = await app.inject({ method: "POST", url: `/v1/chats/${chat.id}/fork`, payload: {} });
|
|
assert.equal(fork.json().chat.reasoningEffort, "high");
|
|
const patch = async (payload: object) => app.inject({ method: "PATCH", url: `/v1/chats/${chat.id}`, payload });
|
|
assert.equal((await patch({ title: "Renamed" })).json().chat.reasoningEffort, "high");
|
|
const switched = (await patch({ provider: "openai", model: "gpt-5.1" })).json().chat;
|
|
assert.equal(switched.lastUsedModel, "gpt-5.1");
|
|
assert.equal(switched.reasoningEffort, null);
|
|
assert.equal((await patch({ reasoningEffort: "xhigh" })).json().chat.reasoningEffort, null);
|
|
assert.equal((await patch({ reasoningEffort: "low" })).json().chat.reasoningEffort, "low");
|
|
assert.equal((await patch({ reasoningEffort: null })).json().chat.reasoningEffort, null);
|
|
assert.equal((await patch({ model: "gpt-5.2" })).statusCode, 400);
|
|
assert.equal((await patch({ reasoningEffort: "bogus" })).statusCode, 400);
|
|
const catalog = (await app.inject({ method: "GET", url: "/v1/models" })).json();
|
|
assert.deepEqual(catalog.providers.openai.reasoningCapabilities["gpt-5.2"].efforts, ["none", "low", "medium", "high", "xhigh"]);
|
|
});
|
|
|
|
test("saved effort is inherited by older clients, overridden/reset explicitly, and never carried to another model", async () => {
|
|
for (const streaming of [false, true]) {
|
|
const chat = await prisma.chat.create({ data: { lastUsedProvider: "openai", lastUsedModel: "gpt-5.2", reasoningEffort: "high", enabledTools: [] } });
|
|
const send = async (extra: object) => {
|
|
const response = await app.inject({ method: "POST", url: `/v1/chat-completions${streaming ? "/stream" : ""}`,
|
|
payload: { chatId: chat.id, provider: "openai", model: "gpt-5.2", messages: [{ role: "user", content: "Hello" }], ...extra } });
|
|
assert.equal(response.statusCode, 200, response.body);
|
|
assert.doesNotMatch(response.body, /event: error/);
|
|
return requests.at(-1)!.body;
|
|
};
|
|
assert.equal((await send({})).reasoning_effort, "high");
|
|
assert.equal((await send({ reasoningEffort: "low" })).reasoning_effort, "low");
|
|
assert.equal((await prisma.chat.findUniqueOrThrow({ where: { id: chat.id } })).reasoningEffort, "low");
|
|
assert.equal((await send({ reasoningEffort: null })).reasoning_effort, undefined);
|
|
await send({ reasoningEffort: "high" });
|
|
assert.equal((await send({ model: "gpt-5.1" })).reasoning_effort, undefined);
|
|
assert.equal((await prisma.chat.findUniqueOrThrow({ where: { id: chat.id } })).reasoningEffort, null);
|
|
}
|
|
});
|
|
|
|
test("Quick Questions accept validated effort without inheriting or persisting chat settings", async () => {
|
|
const count = await prisma.chat.count();
|
|
for (const [model, effort, expected] of [["gpt-5.2", "high", "high"], ["gpt-5.1", "xhigh", undefined], ["chat-latest", "high", undefined], ["gpt-5.2", undefined, undefined]] as const) {
|
|
const response = await app.inject({ method: "POST", url: "/v1/quick-questions/stream", payload: { provider: "openai", model, question: "Hello", enabledTools: [], reasoningEffort: effort } });
|
|
assert.equal(response.statusCode, 200, response.body);
|
|
assert.doesNotMatch(response.body, /event: error/);
|
|
assert.equal(requests.at(-1)!.body.reasoning_effort, expected);
|
|
}
|
|
assert.equal(await prisma.chat.count(), count);
|
|
});
|