feat: add persistent model-aware reasoning effort settings
This commit is contained in:
@@ -12,7 +12,8 @@ import { searchBrave } from "../search/brave.js";
|
||||
import { exaClient } from "../search/exa.js";
|
||||
import { searchSearxng } from "../search/searxng.js";
|
||||
import { extractPdfText, hasPdfMagic, MAX_PDF_PAGES_PER_FETCH } from "./pdf-text.js";
|
||||
import type { ChatMessage } from "./types.js";
|
||||
import type { ChatMessage, Provider } from "./types.js";
|
||||
import type { ReasoningEffort } from "./reasoning-effort.js";
|
||||
|
||||
export const MAX_TOOL_ROUNDS = env.CHAT_MAX_TOOL_ROUNDS;
|
||||
const DEFAULT_WEB_RESULTS = 5;
|
||||
@@ -270,12 +271,14 @@ export type ToolAwareStreamingEvent =
|
||||
|
||||
export type ToolAwareCompletionParams = {
|
||||
client: any;
|
||||
provider?: Provider;
|
||||
model: string;
|
||||
messages: ChatMessage[];
|
||||
enabledTools?: string[];
|
||||
userLocation?: string;
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
reasoningEffort?: ReasoningEffort | null;
|
||||
onToolEvent?: (event: ToolExecutionEvent) => void | Promise<void>;
|
||||
logContext?: {
|
||||
provider: string;
|
||||
|
||||
@@ -4,11 +4,13 @@ import {
|
||||
listModelCatalogProviders,
|
||||
} from "./provider-adapters.js";
|
||||
import type { Provider } from "./types.js";
|
||||
import { getReasoningCatalog, type ReasoningCapabilities } from "./reasoning-effort.js";
|
||||
|
||||
export type ProviderModelSnapshot = {
|
||||
models: string[];
|
||||
loadedAt: string | null;
|
||||
error: string | null;
|
||||
reasoningCapabilities?: Record<string, ReasoningCapabilities>;
|
||||
};
|
||||
|
||||
export type ModelCatalogSnapshot = Partial<Record<Provider, ProviderModelSnapshot>>;
|
||||
@@ -92,6 +94,7 @@ export function getModelCatalogSnapshot(): ModelCatalogSnapshot {
|
||||
models: [...entry.models],
|
||||
loadedAt: entry.loadedAt,
|
||||
error: entry.error,
|
||||
reasoningCapabilities: getReasoningCatalog(provider),
|
||||
};
|
||||
}
|
||||
return snapshot;
|
||||
|
||||
@@ -3,8 +3,10 @@ import { prisma } from "../db.js";
|
||||
import { buildToolLogMessageData } from "./chat-tools.js";
|
||||
import { getProviderChatAdapter } from "./provider-adapters.js";
|
||||
import type { MultiplexRequest, MultiplexResponse } from "./types.js";
|
||||
import { normalizeReasoningEffort } from "./reasoning-effort.js";
|
||||
|
||||
export async function runMultiplex(req: MultiplexRequest): Promise<MultiplexResponse> {
|
||||
req = { ...req, reasoningEffort: normalizeReasoningEffort(req.provider, req.model, req.reasoningEffort) };
|
||||
const t0 = performance.now();
|
||||
const chatId = req.chatId ?? (await prisma.chat.create({ data: {}, select: { id: true } })).id;
|
||||
|
||||
@@ -25,6 +27,7 @@ export async function runMultiplex(req: MultiplexRequest): Promise<MultiplexResp
|
||||
data: {
|
||||
lastUsedProvider: req.provider,
|
||||
lastUsedModel: req.model,
|
||||
reasoningEffort: req.reasoningEffort,
|
||||
},
|
||||
}),
|
||||
prisma.chat.updateMany({
|
||||
@@ -49,6 +52,7 @@ export async function runMultiplex(req: MultiplexRequest): Promise<MultiplexResp
|
||||
userLocation: req.userLocation,
|
||||
temperature: req.temperature,
|
||||
maxTokens: req.maxTokens,
|
||||
reasoningEffort: req.reasoningEffort,
|
||||
logContext: {
|
||||
provider: req.provider,
|
||||
model: req.model,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { getReasoningCapabilities, reasoningRequestOptions } from "../reasoning-effort.js";
|
||||
import {
|
||||
appendDanglingToolIntentCorrection,
|
||||
buildChatToolSystemPrompt,
|
||||
@@ -114,7 +115,8 @@ export async function completeWithChatCompletionsApi(params: ToolAwareCompletion
|
||||
model: params.model,
|
||||
messages: normalizePlainMessages(params.messages, params.userLocation),
|
||||
temperature: params.temperature,
|
||||
max_tokens: params.maxTokens,
|
||||
...(getReasoningCapabilities(params.provider, params.model) ? { max_completion_tokens: params.maxTokens } : { max_tokens: params.maxTokens }),
|
||||
...reasoningRequestOptions(params, "chat-completions"),
|
||||
} as any);
|
||||
|
||||
const usageAcc: Required<NonNullable<ToolAwareCompletionResult["usage"]>> = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
|
||||
@@ -142,7 +144,8 @@ export async function completeWithChatCompletionsApi(params: ToolAwareCompletion
|
||||
model: params.model,
|
||||
messages: conversation,
|
||||
temperature: params.temperature,
|
||||
max_tokens: params.maxTokens,
|
||||
...(getReasoningCapabilities(params.provider, params.model) ? { max_completion_tokens: params.maxTokens } : { max_tokens: params.maxTokens }),
|
||||
...reasoningRequestOptions(params, "chat-completions"),
|
||||
tools: enabledTools,
|
||||
tool_choice: "auto",
|
||||
} as any);
|
||||
@@ -227,7 +230,8 @@ export async function* streamWithChatCompletionsApi(params: ToolAwareCompletionP
|
||||
model: params.model,
|
||||
messages: normalizePlainMessages(params.messages, params.userLocation),
|
||||
temperature: params.temperature,
|
||||
max_tokens: params.maxTokens,
|
||||
...(getReasoningCapabilities(params.provider, params.model) ? { max_completion_tokens: params.maxTokens } : { max_tokens: params.maxTokens }),
|
||||
...reasoningRequestOptions(params, "chat-completions"),
|
||||
stream: true,
|
||||
} as any);
|
||||
|
||||
@@ -267,7 +271,8 @@ export async function* streamWithChatCompletionsApi(params: ToolAwareCompletionP
|
||||
model: params.model,
|
||||
messages: conversation,
|
||||
temperature: params.temperature,
|
||||
max_tokens: params.maxTokens,
|
||||
...(getReasoningCapabilities(params.provider, params.model) ? { max_completion_tokens: params.maxTokens } : { max_tokens: params.maxTokens }),
|
||||
...reasoningRequestOptions(params, "chat-completions"),
|
||||
tools: enabledTools,
|
||||
tool_choice: "auto",
|
||||
stream: true,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { reasoningRequestOptions } from "../reasoning-effort.js";
|
||||
import {
|
||||
appendDanglingToolIntentCorrection,
|
||||
buildChatToolSystemPrompt,
|
||||
@@ -155,6 +156,7 @@ export async function completeWithResponsesApi(params: ToolAwareCompletionParams
|
||||
input,
|
||||
temperature: params.temperature,
|
||||
max_output_tokens: params.maxTokens,
|
||||
...reasoningRequestOptions(params, "responses"),
|
||||
tools: toResponsesTools(enabledTools),
|
||||
tool_choice: "auto",
|
||||
parallel_tool_calls: true,
|
||||
@@ -225,6 +227,7 @@ export async function* streamWithResponsesApi(params: ToolAwareCompletionParams)
|
||||
input,
|
||||
temperature: params.temperature,
|
||||
max_output_tokens: params.maxTokens,
|
||||
...reasoningRequestOptions(params, "responses"),
|
||||
tools: toResponsesTools(enabledTools),
|
||||
tool_choice: "auto",
|
||||
parallel_tool_calls: true,
|
||||
|
||||
@@ -10,6 +10,7 @@ import { completeWithMessagesApi, streamWithMessagesApi } from "./protocols/mess
|
||||
import { completeWithResponsesApi, streamWithResponsesApi } from "./protocols/responses-api.js";
|
||||
import { anthropicClient, geminiClient, openaiClient, xaiClient } from "./providers.js";
|
||||
import type { ChatMessage, Provider } from "./types.js";
|
||||
import { normalizeReasoningEffort, type ReasoningEffort } from "./reasoning-effort.js";
|
||||
|
||||
type ProviderAdapterParams = {
|
||||
model: string;
|
||||
@@ -18,6 +19,7 @@ type ProviderAdapterParams = {
|
||||
userLocation?: string;
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
reasoningEffort?: ReasoningEffort | null;
|
||||
logContext?: ToolAwareCompletionParams["logContext"];
|
||||
};
|
||||
|
||||
@@ -126,15 +128,19 @@ async function fetchJson(url: URL): Promise<any> {
|
||||
return body;
|
||||
}
|
||||
|
||||
function withClient(params: ProviderAdapterParams, client: any, enabledTools?: string[]): ToolAwareCompletionParams {
|
||||
function withClient(params: ProviderAdapterParams, provider: Provider, client: any, enabledTools?: string[]): ToolAwareCompletionParams {
|
||||
const reasoningEffort = normalizeReasoningEffort(provider, params.model, params.reasoningEffort);
|
||||
return {
|
||||
client,
|
||||
provider,
|
||||
model: params.model,
|
||||
messages: params.messages,
|
||||
enabledTools,
|
||||
userLocation: params.userLocation,
|
||||
temperature: params.temperature,
|
||||
// OpenAI reasoning efforts other than none do not accept temperature.
|
||||
temperature: reasoningEffort && reasoningEffort !== "none" ? undefined : params.temperature,
|
||||
maxTokens: params.maxTokens,
|
||||
reasoningEffort,
|
||||
logContext: params.logContext,
|
||||
};
|
||||
}
|
||||
@@ -154,11 +160,11 @@ function createProviderChatAdapter(provider: Provider, spec: ProviderBackendSpec
|
||||
provider,
|
||||
complete(params) {
|
||||
const selected = selectChatProtocol(spec, params);
|
||||
return selected.protocol.complete(withClient(params, spec.createClient(), selected.enabledTools));
|
||||
return selected.protocol.complete(withClient(params, provider, spec.createClient(), selected.enabledTools));
|
||||
},
|
||||
stream(params) {
|
||||
const selected = selectChatProtocol(spec, params);
|
||||
return selected.protocol.stream(withClient(params, spec.createClient(), selected.enabledTools));
|
||||
return selected.protocol.stream(withClient(params, provider, spec.createClient(), selected.enabledTools));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ export type QuickQuestionRequest = {
|
||||
userLocation?: string;
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
reasoningEffort?: MultiplexRequest["reasoningEffort"];
|
||||
};
|
||||
|
||||
export function buildQuickQuestionMultiplexRequest({
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
export const REASONING_EFFORTS = ["none", "minimal", "low", "medium", "high", "xhigh", "max"] as const;
|
||||
export type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
|
||||
export type ReasoningCapabilities = {
|
||||
efforts: ReasoningEffort[];
|
||||
defaultEffort: ReasoningEffort | null;
|
||||
};
|
||||
|
||||
// Verified against official model pages on 2026-11-15:
|
||||
// https://developers.openai.com/api/docs/models/{model}
|
||||
// https://developers.openai.com/api/docs/guides/deployment-checklist#set-up-reasoningeffort
|
||||
// Use exact documented aliases/snapshots. Do not infer support from a prefix:
|
||||
// chat-latest, pro/codex variants, fine-tunes, and future snapshots can differ.
|
||||
const modelGroups: { models: string[]; capabilities: ReasoningCapabilities }[] = [
|
||||
{
|
||||
models: ["gpt-5", "gpt-5-2025-08-07"],
|
||||
capabilities: { efforts: ["minimal", "low", "medium", "high"], defaultEffort: "medium" },
|
||||
},
|
||||
{
|
||||
models: ["gpt-5.1", "gpt-5.1-2025-11-13"],
|
||||
capabilities: { efforts: ["none", "low", "medium", "high"], defaultEffort: "none" },
|
||||
},
|
||||
{
|
||||
models: ["gpt-5.2", "gpt-5.2-2025-12-11", "gpt-5.4", "gpt-5.4-2026-03-05",
|
||||
"gpt-5.4-mini", "gpt-5.4-mini-2026-03-17", "gpt-5.4-nano", "gpt-5.4-nano-2026-03-17"],
|
||||
capabilities: { efforts: ["none", "low", "medium", "high", "xhigh"], defaultEffort: "none" },
|
||||
},
|
||||
{
|
||||
models: ["gpt-5.5", "gpt-5.5-2026-04-23"],
|
||||
capabilities: { efforts: ["none", "low", "medium", "high", "xhigh"], defaultEffort: "medium" },
|
||||
},
|
||||
{
|
||||
models: ["gpt-5.6", "gpt-5.6-sol", "gpt-6-sol", "gpt-6-luna"],
|
||||
capabilities: { efforts: ["none", "low", "medium", "high", "xhigh", "max"], defaultEffort: "medium" },
|
||||
},
|
||||
{
|
||||
models: ["gpt-6.1-sol"],
|
||||
capabilities: { efforts: ["low", "medium", "high", "xhigh", "max"], defaultEffort: "medium" },
|
||||
},
|
||||
{
|
||||
models: ["gpt-6-astra"],
|
||||
capabilities: { efforts: ["low", "medium", "high", "xhigh", "max"], defaultEffort: null },
|
||||
},
|
||||
];
|
||||
|
||||
const openAIModels = new Map(modelGroups.flatMap(({ models, capabilities }) => models.map((model) => [model, capabilities] as const)));
|
||||
|
||||
export function getReasoningCapabilities(provider: string | null | undefined, model: string | null | undefined) {
|
||||
return provider === "openai" && model ? openAIModels.get(model) ?? null : null;
|
||||
}
|
||||
|
||||
export function getReasoningCatalog(provider: string): Record<string, ReasoningCapabilities> {
|
||||
if (provider !== "openai") return {};
|
||||
return Object.fromEntries([...openAIModels].map(([model, value]) => [model, { ...value, efforts: [...value.efforts] }]));
|
||||
}
|
||||
|
||||
export function normalizeReasoningEffort(provider: string | null | undefined, model: string | null | undefined, effort: unknown): ReasoningEffort | null {
|
||||
const capabilities = getReasoningCapabilities(provider, model);
|
||||
return capabilities?.efforts.includes(effort as ReasoningEffort) ? effort as ReasoningEffort : null;
|
||||
}
|
||||
|
||||
export function reasoningRequestOptions(params: { provider?: string; model: string; reasoningEffort?: ReasoningEffort | null }, protocol: "responses" | "chat-completions") {
|
||||
const effort = normalizeReasoningEffort(params.provider, params.model, params.reasoningEffort);
|
||||
if (!effort) return {};
|
||||
return protocol === "responses" ? { reasoning: { effort } } : { reasoning_effort: effort };
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
} from "./chat-tools.js";
|
||||
import { getProviderChatAdapter } from "./provider-adapters.js";
|
||||
import type { MultiplexRequest, Provider } from "./types.js";
|
||||
import { normalizeReasoningEffort } from "./reasoning-effort.js";
|
||||
|
||||
type StreamUsage = {
|
||||
inputTokens?: number;
|
||||
@@ -26,6 +27,7 @@ function getChatIdOrCreate(chatId?: string) {
|
||||
}
|
||||
|
||||
export async function* runMultiplexStream(req: MultiplexRequest): AsyncGenerator<StreamEvent> {
|
||||
req = { ...req, reasoningEffort: normalizeReasoningEffort(req.provider, req.model, req.reasoningEffort) };
|
||||
const t0 = performance.now();
|
||||
const shouldPersist = req.persist !== false;
|
||||
const chatId = shouldPersist ? await getChatIdOrCreate(req.chatId) : null;
|
||||
@@ -50,6 +52,7 @@ export async function* runMultiplexStream(req: MultiplexRequest): AsyncGenerator
|
||||
data: {
|
||||
lastUsedProvider: req.provider,
|
||||
lastUsedModel: req.model,
|
||||
reasoningEffort: req.reasoningEffort,
|
||||
},
|
||||
}),
|
||||
prisma.chat.updateMany({
|
||||
@@ -77,6 +80,7 @@ export async function* runMultiplexStream(req: MultiplexRequest): AsyncGenerator
|
||||
userLocation: req.userLocation,
|
||||
temperature: req.temperature,
|
||||
maxTokens: req.maxTokens,
|
||||
reasoningEffort: req.reasoningEffort,
|
||||
logContext: {
|
||||
provider: req.provider,
|
||||
model: req.model,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { ReasoningEffort } from "./reasoning-effort.js";
|
||||
|
||||
export const PROVIDERS = ["openai", "anthropic", "xai", "gemini"] as const;
|
||||
|
||||
export type Provider = (typeof PROVIDERS)[number];
|
||||
@@ -42,6 +44,7 @@ export type MultiplexRequest = {
|
||||
userLocation?: string;
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
reasoningEffort?: ReasoningEffort | null;
|
||||
};
|
||||
|
||||
export type MultiplexResponse = {
|
||||
|
||||
Reference in New Issue
Block a user