feat: add persistent model-aware reasoning effort settings

This commit is contained in:
2026-11-15 18:45:22 -08:00
parent 91c7c0310e
commit 92a16a39c0
22 changed files with 678 additions and 36 deletions
+4 -1
View File
@@ -12,7 +12,8 @@ import { searchBrave } from "../search/brave.js";
import { exaClient } from "../search/exa.js";
import { searchSearxng } from "../search/searxng.js";
import { extractPdfText, hasPdfMagic, MAX_PDF_PAGES_PER_FETCH } from "./pdf-text.js";
import type { ChatMessage } from "./types.js";
import type { ChatMessage, Provider } from "./types.js";
import type { ReasoningEffort } from "./reasoning-effort.js";
export const MAX_TOOL_ROUNDS = env.CHAT_MAX_TOOL_ROUNDS;
const DEFAULT_WEB_RESULTS = 5;
@@ -270,12 +271,14 @@ export type ToolAwareStreamingEvent =
export type ToolAwareCompletionParams = {
client: any;
provider?: Provider;
model: string;
messages: ChatMessage[];
enabledTools?: string[];
userLocation?: string;
temperature?: number;
maxTokens?: number;
reasoningEffort?: ReasoningEffort | null;
onToolEvent?: (event: ToolExecutionEvent) => void | Promise<void>;
logContext?: {
provider: string;
+3
View File
@@ -4,11 +4,13 @@ import {
listModelCatalogProviders,
} from "./provider-adapters.js";
import type { Provider } from "./types.js";
import { getReasoningCatalog, type ReasoningCapabilities } from "./reasoning-effort.js";
export type ProviderModelSnapshot = {
models: string[];
loadedAt: string | null;
error: string | null;
reasoningCapabilities?: Record<string, ReasoningCapabilities>;
};
export type ModelCatalogSnapshot = Partial<Record<Provider, ProviderModelSnapshot>>;
@@ -92,6 +94,7 @@ export function getModelCatalogSnapshot(): ModelCatalogSnapshot {
models: [...entry.models],
loadedAt: entry.loadedAt,
error: entry.error,
reasoningCapabilities: getReasoningCatalog(provider),
};
}
return snapshot;
+4
View File
@@ -3,8 +3,10 @@ import { prisma } from "../db.js";
import { buildToolLogMessageData } from "./chat-tools.js";
import { getProviderChatAdapter } from "./provider-adapters.js";
import type { MultiplexRequest, MultiplexResponse } from "./types.js";
import { normalizeReasoningEffort } from "./reasoning-effort.js";
export async function runMultiplex(req: MultiplexRequest): Promise<MultiplexResponse> {
req = { ...req, reasoningEffort: normalizeReasoningEffort(req.provider, req.model, req.reasoningEffort) };
const t0 = performance.now();
const chatId = req.chatId ?? (await prisma.chat.create({ data: {}, select: { id: true } })).id;
@@ -25,6 +27,7 @@ export async function runMultiplex(req: MultiplexRequest): Promise<MultiplexResp
data: {
lastUsedProvider: req.provider,
lastUsedModel: req.model,
reasoningEffort: req.reasoningEffort,
},
}),
prisma.chat.updateMany({
@@ -49,6 +52,7 @@ export async function runMultiplex(req: MultiplexRequest): Promise<MultiplexResp
userLocation: req.userLocation,
temperature: req.temperature,
maxTokens: req.maxTokens,
reasoningEffort: req.reasoningEffort,
logContext: {
provider: req.provider,
model: req.model,
@@ -1,3 +1,4 @@
import { getReasoningCapabilities, reasoningRequestOptions } from "../reasoning-effort.js";
import {
appendDanglingToolIntentCorrection,
buildChatToolSystemPrompt,
@@ -114,7 +115,8 @@ export async function completeWithChatCompletionsApi(params: ToolAwareCompletion
model: params.model,
messages: normalizePlainMessages(params.messages, params.userLocation),
temperature: params.temperature,
max_tokens: params.maxTokens,
...(getReasoningCapabilities(params.provider, params.model) ? { max_completion_tokens: params.maxTokens } : { max_tokens: params.maxTokens }),
...reasoningRequestOptions(params, "chat-completions"),
} as any);
const usageAcc: Required<NonNullable<ToolAwareCompletionResult["usage"]>> = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
@@ -142,7 +144,8 @@ export async function completeWithChatCompletionsApi(params: ToolAwareCompletion
model: params.model,
messages: conversation,
temperature: params.temperature,
max_tokens: params.maxTokens,
...(getReasoningCapabilities(params.provider, params.model) ? { max_completion_tokens: params.maxTokens } : { max_tokens: params.maxTokens }),
...reasoningRequestOptions(params, "chat-completions"),
tools: enabledTools,
tool_choice: "auto",
} as any);
@@ -227,7 +230,8 @@ export async function* streamWithChatCompletionsApi(params: ToolAwareCompletionP
model: params.model,
messages: normalizePlainMessages(params.messages, params.userLocation),
temperature: params.temperature,
max_tokens: params.maxTokens,
...(getReasoningCapabilities(params.provider, params.model) ? { max_completion_tokens: params.maxTokens } : { max_tokens: params.maxTokens }),
...reasoningRequestOptions(params, "chat-completions"),
stream: true,
} as any);
@@ -267,7 +271,8 @@ export async function* streamWithChatCompletionsApi(params: ToolAwareCompletionP
model: params.model,
messages: conversation,
temperature: params.temperature,
max_tokens: params.maxTokens,
...(getReasoningCapabilities(params.provider, params.model) ? { max_completion_tokens: params.maxTokens } : { max_tokens: params.maxTokens }),
...reasoningRequestOptions(params, "chat-completions"),
tools: enabledTools,
tool_choice: "auto",
stream: true,
@@ -1,3 +1,4 @@
import { reasoningRequestOptions } from "../reasoning-effort.js";
import {
appendDanglingToolIntentCorrection,
buildChatToolSystemPrompt,
@@ -155,6 +156,7 @@ export async function completeWithResponsesApi(params: ToolAwareCompletionParams
input,
temperature: params.temperature,
max_output_tokens: params.maxTokens,
...reasoningRequestOptions(params, "responses"),
tools: toResponsesTools(enabledTools),
tool_choice: "auto",
parallel_tool_calls: true,
@@ -225,6 +227,7 @@ export async function* streamWithResponsesApi(params: ToolAwareCompletionParams)
input,
temperature: params.temperature,
max_output_tokens: params.maxTokens,
...reasoningRequestOptions(params, "responses"),
tools: toResponsesTools(enabledTools),
tool_choice: "auto",
parallel_tool_calls: true,
+10 -4
View File
@@ -10,6 +10,7 @@ import { completeWithMessagesApi, streamWithMessagesApi } from "./protocols/mess
import { completeWithResponsesApi, streamWithResponsesApi } from "./protocols/responses-api.js";
import { anthropicClient, geminiClient, openaiClient, xaiClient } from "./providers.js";
import type { ChatMessage, Provider } from "./types.js";
import { normalizeReasoningEffort, type ReasoningEffort } from "./reasoning-effort.js";
type ProviderAdapterParams = {
model: string;
@@ -18,6 +19,7 @@ type ProviderAdapterParams = {
userLocation?: string;
temperature?: number;
maxTokens?: number;
reasoningEffort?: ReasoningEffort | null;
logContext?: ToolAwareCompletionParams["logContext"];
};
@@ -126,15 +128,19 @@ async function fetchJson(url: URL): Promise<any> {
return body;
}
function withClient(params: ProviderAdapterParams, client: any, enabledTools?: string[]): ToolAwareCompletionParams {
function withClient(params: ProviderAdapterParams, provider: Provider, client: any, enabledTools?: string[]): ToolAwareCompletionParams {
const reasoningEffort = normalizeReasoningEffort(provider, params.model, params.reasoningEffort);
return {
client,
provider,
model: params.model,
messages: params.messages,
enabledTools,
userLocation: params.userLocation,
temperature: params.temperature,
// OpenAI reasoning efforts other than none do not accept temperature.
temperature: reasoningEffort && reasoningEffort !== "none" ? undefined : params.temperature,
maxTokens: params.maxTokens,
reasoningEffort,
logContext: params.logContext,
};
}
@@ -154,11 +160,11 @@ function createProviderChatAdapter(provider: Provider, spec: ProviderBackendSpec
provider,
complete(params) {
const selected = selectChatProtocol(spec, params);
return selected.protocol.complete(withClient(params, spec.createClient(), selected.enabledTools));
return selected.protocol.complete(withClient(params, provider, spec.createClient(), selected.enabledTools));
},
stream(params) {
const selected = selectChatProtocol(spec, params);
return selected.protocol.stream(withClient(params, spec.createClient(), selected.enabledTools));
return selected.protocol.stream(withClient(params, provider, spec.createClient(), selected.enabledTools));
},
};
}
+1
View File
@@ -13,6 +13,7 @@ export type QuickQuestionRequest = {
userLocation?: string;
temperature?: number;
maxTokens?: number;
reasoningEffort?: MultiplexRequest["reasoningEffort"];
};
export function buildQuickQuestionMultiplexRequest({
+65
View File
@@ -0,0 +1,65 @@
export const REASONING_EFFORTS = ["none", "minimal", "low", "medium", "high", "xhigh", "max"] as const;
export type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
export type ReasoningCapabilities = {
efforts: ReasoningEffort[];
defaultEffort: ReasoningEffort | null;
};
// Verified against official model pages on 2026-11-15:
// https://developers.openai.com/api/docs/models/{model}
// https://developers.openai.com/api/docs/guides/deployment-checklist#set-up-reasoningeffort
// Use exact documented aliases/snapshots. Do not infer support from a prefix:
// chat-latest, pro/codex variants, fine-tunes, and future snapshots can differ.
const modelGroups: { models: string[]; capabilities: ReasoningCapabilities }[] = [
{
models: ["gpt-5", "gpt-5-2025-08-07"],
capabilities: { efforts: ["minimal", "low", "medium", "high"], defaultEffort: "medium" },
},
{
models: ["gpt-5.1", "gpt-5.1-2025-11-13"],
capabilities: { efforts: ["none", "low", "medium", "high"], defaultEffort: "none" },
},
{
models: ["gpt-5.2", "gpt-5.2-2025-12-11", "gpt-5.4", "gpt-5.4-2026-03-05",
"gpt-5.4-mini", "gpt-5.4-mini-2026-03-17", "gpt-5.4-nano", "gpt-5.4-nano-2026-03-17"],
capabilities: { efforts: ["none", "low", "medium", "high", "xhigh"], defaultEffort: "none" },
},
{
models: ["gpt-5.5", "gpt-5.5-2026-04-23"],
capabilities: { efforts: ["none", "low", "medium", "high", "xhigh"], defaultEffort: "medium" },
},
{
models: ["gpt-5.6", "gpt-5.6-sol", "gpt-6-sol", "gpt-6-luna"],
capabilities: { efforts: ["none", "low", "medium", "high", "xhigh", "max"], defaultEffort: "medium" },
},
{
models: ["gpt-6.1-sol"],
capabilities: { efforts: ["low", "medium", "high", "xhigh", "max"], defaultEffort: "medium" },
},
{
models: ["gpt-6-astra"],
capabilities: { efforts: ["low", "medium", "high", "xhigh", "max"], defaultEffort: null },
},
];
const openAIModels = new Map(modelGroups.flatMap(({ models, capabilities }) => models.map((model) => [model, capabilities] as const)));
export function getReasoningCapabilities(provider: string | null | undefined, model: string | null | undefined) {
return provider === "openai" && model ? openAIModels.get(model) ?? null : null;
}
export function getReasoningCatalog(provider: string): Record<string, ReasoningCapabilities> {
if (provider !== "openai") return {};
return Object.fromEntries([...openAIModels].map(([model, value]) => [model, { ...value, efforts: [...value.efforts] }]));
}
export function normalizeReasoningEffort(provider: string | null | undefined, model: string | null | undefined, effort: unknown): ReasoningEffort | null {
const capabilities = getReasoningCapabilities(provider, model);
return capabilities?.efforts.includes(effort as ReasoningEffort) ? effort as ReasoningEffort : null;
}
export function reasoningRequestOptions(params: { provider?: string; model: string; reasoningEffort?: ReasoningEffort | null }, protocol: "responses" | "chat-completions") {
const effort = normalizeReasoningEffort(params.provider, params.model, params.reasoningEffort);
if (!effort) return {};
return protocol === "responses" ? { reasoning: { effort } } : { reasoning_effort: effort };
}
+4
View File
@@ -6,6 +6,7 @@ import {
} from "./chat-tools.js";
import { getProviderChatAdapter } from "./provider-adapters.js";
import type { MultiplexRequest, Provider } from "./types.js";
import { normalizeReasoningEffort } from "./reasoning-effort.js";
type StreamUsage = {
inputTokens?: number;
@@ -26,6 +27,7 @@ function getChatIdOrCreate(chatId?: string) {
}
export async function* runMultiplexStream(req: MultiplexRequest): AsyncGenerator<StreamEvent> {
req = { ...req, reasoningEffort: normalizeReasoningEffort(req.provider, req.model, req.reasoningEffort) };
const t0 = performance.now();
const shouldPersist = req.persist !== false;
const chatId = shouldPersist ? await getChatIdOrCreate(req.chatId) : null;
@@ -50,6 +52,7 @@ export async function* runMultiplexStream(req: MultiplexRequest): AsyncGenerator
data: {
lastUsedProvider: req.provider,
lastUsedModel: req.model,
reasoningEffort: req.reasoningEffort,
},
}),
prisma.chat.updateMany({
@@ -77,6 +80,7 @@ export async function* runMultiplexStream(req: MultiplexRequest): AsyncGenerator
userLocation: req.userLocation,
temperature: req.temperature,
maxTokens: req.maxTokens,
reasoningEffort: req.reasoningEffort,
logContext: {
provider: req.provider,
model: req.model,
+3
View File
@@ -1,3 +1,5 @@
import type { ReasoningEffort } from "./reasoning-effort.js";
export const PROVIDERS = ["openai", "anthropic", "xai", "gemini"] as const;
export type Provider = (typeof PROVIDERS)[number];
@@ -42,6 +44,7 @@ export type MultiplexRequest = {
userLocation?: string;
temperature?: number;
maxTokens?: number;
reasoningEffort?: ReasoningEffort | null;
};
export type MultiplexResponse = {
+41 -11
View File
@@ -16,6 +16,7 @@ import { runMultiplexStream, type StreamEvent } from "./llm/streaming.js";
import { getAvailableChatTools, normalizeEnabledChatTools } from "./llm/chat-tools.js";
import { getModelCatalogSnapshot } from "./llm/model-catalog.js";
import { buildQuickQuestionMultiplexRequest } from "./llm/quick-question.js";
import { normalizeReasoningEffort, REASONING_EFFORTS, type ReasoningEffort } from "./llm/reasoning-effort.js";
import { openaiClient } from "./llm/providers.js";
import { serializeProviderFields } from "./llm/provider-ids.js";
import { exaClient } from "./search/exa.js";
@@ -24,6 +25,7 @@ import { searchWorkspace } from "./workspace-search.js";
import { PROVIDERS, type ChatAttachment } from "./llm/types.js";
const ProviderSchema = z.enum(PROVIDERS);
const ReasoningEffortSchema = z.enum(REASONING_EFFORTS).nullable();
const MAX_ADDITIONAL_SYSTEM_PROMPT_CHARS = 12_000;
const MAX_FORK_MESSAGE_SNIPPET_CHARS = 48;
const EnabledToolsSchema = z.array(z.string().trim().min(1).max(80)).max(20).transform((value) => normalizeEnabledChatTools(value));
@@ -209,6 +211,7 @@ const CompletionStreamBody = z
userLocation: z.string().trim().min(1).max(200).optional(),
temperature: z.number().min(0).max(2).optional(),
maxTokens: z.number().int().positive().optional(),
reasoningEffort: ReasoningEffortSchema.optional(),
})
.superRefine((value, ctx) => {
if (value.persist === false && value.chatId) {
@@ -235,6 +238,7 @@ const QuickQuestionStreamBody = z.object({
userLocation: z.string().trim().min(1).max(200).optional(),
temperature: z.number().min(0).max(2).optional(),
maxTokens: z.number().int().positive().optional(),
reasoningEffort: ReasoningEffortSchema.optional(),
});
function mergeAttachmentsIntoMetadata(metadata: unknown, attachments?: ChatAttachment[]) {
@@ -269,16 +273,16 @@ function prependAdditionalSystemPrompt<T extends { messages: IncomingChatMessage
};
}
async function applyStoredChatSettings<T extends { chatId?: string; messages: IncomingChatMessage[]; additionalSystemPrompt?: string; enabledTools?: string[] }>(
async function applyStoredChatSettings<T extends { chatId?: string; provider: string; model: string; messages: IncomingChatMessage[]; additionalSystemPrompt?: string; enabledTools?: string[]; reasoningEffort?: ReasoningEffort | null }>(
body: T
) {
if (!body.chatId || (body.additionalSystemPrompt !== undefined && body.enabledTools !== undefined)) {
if (!body.chatId || (body.additionalSystemPrompt !== undefined && body.enabledTools !== undefined && body.reasoningEffort !== undefined)) {
return prependAdditionalSystemPrompt(body);
}
const chat = await prisma.chat.findUnique({
where: { id: body.chatId },
select: { additionalSystemPrompt: true, enabledTools: true },
select: { additionalSystemPrompt: true, enabledTools: true, reasoningEffort: true, lastUsedProvider: true, lastUsedModel: true },
});
if (!chat) return prependAdditionalSystemPrompt(body);
@@ -286,6 +290,10 @@ async function applyStoredChatSettings<T extends { chatId?: string; messages: In
...body,
additionalSystemPrompt: body.additionalSystemPrompt ?? chat.additionalSystemPrompt ?? undefined,
enabledTools: body.enabledTools ?? normalizeEnabledChatTools(chat.enabledTools),
// Omission inherits only for the saved model; null explicitly resets it.
reasoningEffort: normalizeReasoningEffort(body.provider, body.model, body.reasoningEffort !== undefined
? body.reasoningEffort
: chat.lastUsedProvider === body.provider && chat.lastUsedModel === body.model ? chat.reasoningEffort : null),
});
}
@@ -556,6 +564,7 @@ const chatSummarySelect = {
initiatedModel: true,
lastUsedProvider: true,
lastUsedModel: true,
reasoningEffort: true,
additionalSystemPrompt: true,
enabledTools: true,
projectItems: starredProjectItemsSelect,
@@ -1082,7 +1091,7 @@ export async function registerRoutes(app: FastifyInstance, options: { agentInlin
} : undefined,
orderBy: [{ updatedAt: "desc" }, { id: "asc" }],
take: limit + 1,
select: { id: true, title: true, parentChatId: true, createdAt: true, updatedAt: true, initiatedProvider: true, initiatedModel: true, lastUsedProvider: true, lastUsedModel: true },
select: { id: true, title: true, parentChatId: true, createdAt: true, updatedAt: true, initiatedProvider: true, initiatedModel: true, lastUsedProvider: true, lastUsedModel: true, reasoningEffort: true },
});
const hasMore = chats.length > limit;
const page = hasMore ? chats.slice(0, limit) : chats;
@@ -1103,6 +1112,7 @@ export async function registerRoutes(app: FastifyInstance, options: { agentInlin
title: z.string().optional(),
provider: ProviderSchema.optional(),
model: z.string().trim().min(1).optional(),
reasoningEffort: ReasoningEffortSchema.optional(),
additionalSystemPrompt: z.string().max(MAX_ADDITIONAL_SYSTEM_PROMPT_CHARS).optional(),
enabledTools: EnabledToolsSchema.optional(),
messages: z.array(CompletionMessageSchema).optional(),
@@ -1133,6 +1143,7 @@ export async function registerRoutes(app: FastifyInstance, options: { agentInlin
initiatedModel: body.model,
lastUsedProvider: body.provider,
lastUsedModel: body.model,
reasoningEffort: normalizeReasoningEffort(body.provider, body.model, body.reasoningEffort),
additionalSystemPrompt: normalizeAdditionalSystemPrompt(body.additionalSystemPrompt),
enabledTools: body.enabledTools as any,
messages: body.messages?.length
@@ -1170,6 +1181,7 @@ export async function registerRoutes(app: FastifyInstance, options: { agentInlin
initiatedModel: true,
lastUsedProvider: true,
lastUsedModel: true,
reasoningEffort: true,
additionalSystemPrompt: true,
enabledTools: true,
userId: true,
@@ -1199,6 +1211,7 @@ export async function registerRoutes(app: FastifyInstance, options: { agentInlin
initiatedModel: source.initiatedModel,
lastUsedProvider: source.lastUsedProvider,
lastUsedModel: source.lastUsedModel,
reasoningEffort: normalizeReasoningEffort(source.lastUsedProvider, source.lastUsedModel, source.reasoningEffort),
additionalSystemPrompt: source.additionalSystemPrompt,
enabledTools: (source.enabledTools ?? undefined) as any,
userId: source.userId,
@@ -1231,11 +1244,16 @@ export async function registerRoutes(app: FastifyInstance, options: { agentInlin
const Params = z.object({ chatId: z.string() });
const Body = z.object({
title: z.string().trim().min(1).optional(),
provider: ProviderSchema.optional(),
model: z.string().trim().min(1).optional(),
reasoningEffort: ReasoningEffortSchema.optional(),
additionalSystemPrompt: z.string().max(MAX_ADDITIONAL_SYSTEM_PROMPT_CHARS).nullable().optional(),
enabledTools: EnabledToolsSchema.optional(),
});
}).refine((value) => Boolean(value.provider) === Boolean(value.model), "provider and model must be supplied together");
const { chatId } = Params.parse(req.params);
const body = Body.parse(req.body ?? {});
const parsed = Body.safeParse(req.body ?? {});
if (!parsed.success) return app.httpErrors.badRequest(parsed.error.message);
const body = parsed.data;
const data: Record<string, unknown> = {};
if (body.title !== undefined) {
@@ -1244,13 +1262,24 @@ export async function registerRoutes(app: FastifyInstance, options: { agentInlin
}
if (body.additionalSystemPrompt !== undefined) data.additionalSystemPrompt = normalizeAdditionalSystemPrompt(body.additionalSystemPrompt);
if (body.enabledTools !== undefined) data.enabledTools = body.enabledTools;
const updated = await prisma.chat.updateMany({
where: { id: chatId },
data: data as any,
const updated = await prisma.$transaction(async (tx) => {
const current = await tx.chat.findUnique({ where: { id: chatId } });
if (!current) return null;
const provider = body.provider ?? current.lastUsedProvider;
const model = body.model ?? current.lastUsedModel;
if (body.provider !== undefined) {
data.lastUsedProvider = provider;
data.lastUsedModel = model;
}
if (body.provider !== undefined || body.reasoningEffort !== undefined) {
const sameModel = provider === current.lastUsedProvider && model === current.lastUsedModel;
data.reasoningEffort = normalizeReasoningEffort(provider, model, body.reasoningEffort !== undefined
? body.reasoningEffort : sameModel ? current.reasoningEffort : null);
}
return tx.chat.update({ where: { id: chatId }, data: data as any });
});
if (updated.count === 0) return app.httpErrors.notFound("chat not found");
if (!updated) return app.httpErrors.notFound("chat not found");
const chat = await getChatSummary(chatId);
if (!chat) return app.httpErrors.notFound("chat not found");
@@ -1738,6 +1767,7 @@ export async function registerRoutes(app: FastifyInstance, options: { agentInlin
userLocation: z.string().trim().min(1).max(200).optional(),
temperature: z.number().min(0).max(2).optional(),
maxTokens: z.number().int().positive().optional(),
reasoningEffort: ReasoningEffortSchema.optional(),
});
const parsed = Body.safeParse(req.body);