diff --git a/docs/api/rest.md b/docs/api/rest.md index 809de43..7552a66 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -285,6 +285,7 @@ Behavior notes: - For `chatId` calls, server stores only *new* non-assistant messages from provided history to avoid duplicates. - `additionalSystemPrompt`, when present directly or loaded from stored chat settings, is prepended to the provider request as a `system` message and is not inserted into the persisted chat transcript by this endpoint. - `enabledTools` limits Sybil-managed tools for this request. When omitted for a saved chat, the stored chat setting is used; otherwise all available tools are enabled by default. An empty array disables Sybil-managed tools. +- `maxTokens` is optional. For `anthropic`, when omitted the backend requests the selected model's maximum output token limit from Anthropic's Models API and uses that as `max_tokens`; if the model limit cannot be loaded, the fallback is 128000. For other providers, omitted `maxTokens` is not sent as an explicit cap. - Server persists final assistant output and call metadata (`LlmCall`) in DB. - Server updates chat-level model metadata on each call: `lastUsedProvider`/`lastUsedModel`; first successful/failed call also initializes `initiatedProvider`/`initiatedModel` if unset. - Attachments are optional and currently apply to `user` messages. Persisted chat history stores them under `message.metadata.attachments`. diff --git a/docs/api/streaming-chat.md b/docs/api/streaming-chat.md index 73482cd..84607a0 100644 --- a/docs/api/streaming-chat.md +++ b/docs/api/streaming-chat.md @@ -64,6 +64,7 @@ Notes: - For persisted streams, backend stores only new non-assistant input history rows to avoid duplicates. - `additionalSystemPrompt`, when present directly or loaded from stored chat settings, is prepended to the provider request as a `system` message and is not inserted into the persisted chat transcript by this endpoint. - `enabledTools` limits Sybil-managed tools for this request. When omitted for a saved chat, the stored chat setting is used; otherwise all available tools are enabled by default. An empty array disables Sybil-managed tools. +- `maxTokens` is optional. For `anthropic`, when omitted the backend requests the selected model's maximum output token limit from Anthropic's Models API and uses that as `max_tokens`; if the model limit cannot be loaded, the fallback is 128000. For other providers, omitted `maxTokens` is not sent as an explicit cap. - Attachments are optional and are persisted under `message.metadata.attachments` on stored user messages when `persist` is `true`. Persisted chat streams with a `chatId` are backend-owned active runs: diff --git a/server/src/llm/protocols/messages-api.ts b/server/src/llm/protocols/messages-api.ts index 6052f71..ba5500e 100644 --- a/server/src/llm/protocols/messages-api.ts +++ b/server/src/llm/protocols/messages-api.ts @@ -28,6 +28,45 @@ import type { ChatMessage } from "../types.js"; const INTERNAL_CORRECTION = "Internal correction: the previous assistant message claimed it would run a tool, but no tool call was made. If the task needs an available tool, call it now. Otherwise provide the final answer directly without saying you will run a tool."; +const DEFAULT_ANTHROPIC_MAX_TOKENS = 128_000; +const MODEL_MAX_TOKENS_CACHE_MS = 24 * 60 * 60 * 1000; + +const modelMaxTokensCache = new Map(); + +function readMaxTokens(value: unknown) { + return Number.isSafeInteger(value) && (value as number) > 0 ? (value as number) : undefined; +} + +function getModelInfoMaxTokens(modelInfo: any) { + return readMaxTokens(modelInfo?.max_tokens) ?? readMaxTokens(modelInfo?.maxTokens); +} + +async function getMessagesMaxTokens(params: ToolAwareCompletionParams) { + if (params.maxTokens) return params.maxTokens; + + const cached = modelMaxTokensCache.get(params.model); + if (cached && cached.expiresAt > Date.now()) return cached.maxTokens; + + try { + const retrieve = params.client?.models?.retrieve; + if (typeof retrieve === "function") { + const modelInfo = await retrieve.call(params.client.models, params.model); + const maxTokens = getModelInfoMaxTokens(modelInfo); + if (maxTokens) { + modelMaxTokensCache.set(params.model, { + maxTokens, + expiresAt: Date.now() + MODEL_MAX_TOKENS_CACHE_MS, + }); + return maxTokens; + } + } + } catch { + // Fall back to the documented max for Claude Opus 4.8 and related high-output models. + } + + return DEFAULT_ANTHROPIC_MAX_TOKENS; +} + function toTools(tools: any[]) { return tools .map((tool) => { @@ -160,11 +199,12 @@ function mergeUsage(acc: Required, usage: any) { export async function completeWithMessagesApi(params: ToolAwareCompletionParams): Promise { const enabledTools = getEnabledChatTools(params); + const maxTokens = await getMessagesMaxTokens(params); if (!enabledTools.length) { const response = await params.client.messages.create({ model: params.model, system: buildTopLevelSystemPrompt(params.messages, params.userLocation), - max_tokens: params.maxTokens ?? 1024, + max_tokens: maxTokens, temperature: params.temperature, messages: buildBaseMessages(params), } as any); @@ -192,7 +232,7 @@ export async function completeWithMessagesApi(params: ToolAwareCompletionParams) const response = await params.client.messages.create({ model: params.model, system: buildTopLevelSystemPrompt(params.messages, params.userLocation, buildChatToolSystemPrompt(params)), - max_tokens: params.maxTokens ?? 1024, + max_tokens: maxTokens, temperature: params.temperature, messages: conversation, tools: toTools(enabledTools), @@ -248,6 +288,7 @@ export async function completeWithMessagesApi(params: ToolAwareCompletionParams) export async function* streamWithMessagesApi(params: ToolAwareCompletionParams): AsyncGenerator { const enabledTools = getEnabledChatTools(params); + const maxTokens = await getMessagesMaxTokens(params); if (!enabledTools.length) { const rawResponses: unknown[] = []; const usageAcc: Required = { inputTokens: 0, outputTokens: 0, totalTokens: 0 }; @@ -259,7 +300,7 @@ export async function* streamWithMessagesApi(params: ToolAwareCompletionParams): const stream = await params.client.messages.create({ model: params.model, system: buildTopLevelSystemPrompt(params.messages, params.userLocation), - max_tokens: params.maxTokens ?? 1024, + max_tokens: maxTokens, temperature: params.temperature, messages: buildBaseMessages(params), stream: true, @@ -315,7 +356,7 @@ export async function* streamWithMessagesApi(params: ToolAwareCompletionParams): const stream = await params.client.messages.create({ model: params.model, system: buildTopLevelSystemPrompt(params.messages, params.userLocation, buildChatToolSystemPrompt(params)), - max_tokens: params.maxTokens ?? 1024, + max_tokens: maxTokens, temperature: params.temperature, messages: conversation, tools: toTools(enabledTools), diff --git a/server/tests/chat-tools-streaming.test.ts b/server/tests/chat-tools-streaming.test.ts index aaeab70..d9ff3d0 100644 --- a/server/tests/chat-tools-streaming.test.ts +++ b/server/tests/chat-tools-streaming.test.ts @@ -140,6 +140,94 @@ test("plain Chat Completions stream does not send Sybil-managed tools", async () assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hi"); }); +test("Messages API defaults max_tokens to the Anthropic model maximum", async () => { + let requestBody: any = null; + let retrievedModel: string | null = null; + const client = { + models: { + retrieve: async (model: string) => { + retrievedModel = model; + return { id: model, max_tokens: 128000 }; + }, + }, + messages: { + create: async (body: any) => { + requestBody = body; + return { + content: [{ type: "text", text: "Done" }], + usage: { input_tokens: 1, output_tokens: 1 }, + }; + }, + }, + }; + + const result = await completeWithMessagesApi({ + client: client as any, + model: "claude-max-default-test", + messages: [{ role: "user", content: "Say done" }], + }); + + assert.equal(retrievedModel, "claude-max-default-test"); + assert.equal(requestBody?.max_tokens, 128000); + assert.equal(result.text, "Done"); +}); + +test("Messages API preserves explicit maxTokens", async () => { + let requestBody: any = null; + let didRetrieveModel = false; + const client = { + models: { + retrieve: async () => { + didRetrieveModel = true; + return { max_tokens: 128000 }; + }, + }, + messages: { + create: async (body: any) => { + requestBody = body; + return streamFrom([ + { + type: "message_start", + message: { + usage: { input_tokens: 1, output_tokens: 0 }, + }, + }, + { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: "Done" }, + }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { output_tokens: 1 }, + }, + { type: "message_stop" }, + ]); + }, + }, + }; + + const events = await collectEvents( + streamWithMessagesApi({ + client: client as any, + model: "claude-explicit-max-test", + messages: [{ role: "user", content: "Say done" }], + maxTokens: 4096, + }) + ); + + assert.equal(didRetrieveModel, false); + assert.equal(requestBody?.max_tokens, 4096); + assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Done"); +}); + test("fetch_url sends browser-like navigation headers", async () => { const originalFetch = globalThis.fetch; const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];