Raise Anthropic default max tokens

docs/api/rest.md

@@ -285,6 +285,7 @@ Behavior notes:
 - For `chatId` calls, server stores only *new* non-assistant messages from provided history to avoid duplicates.
 - `additionalSystemPrompt`, when present directly or loaded from stored chat settings, is prepended to the provider request as a `system` message and is not inserted into the persisted chat transcript by this endpoint.
 - `enabledTools` limits Sybil-managed tools for this request. When omitted for a saved chat, the stored chat setting is used; otherwise all available tools are enabled by default. An empty array disables Sybil-managed tools.
+- `maxTokens` is optional. For `anthropic`, when omitted the backend requests the selected model's maximum output token limit from Anthropic's Models API and uses that as `max_tokens`; if the model limit cannot be loaded, the fallback is 128000. For other providers, omitted `maxTokens` is not sent as an explicit cap.
 - Server persists final assistant output and call metadata (`LlmCall`) in DB.
 - Server updates chat-level model metadata on each call: `lastUsedProvider`/`lastUsedModel`; first successful/failed call also initializes `initiatedProvider`/`initiatedModel` if unset.
 - Attachments are optional and currently apply to `user` messages. Persisted chat history stores them under `message.metadata.attachments`.

docs/api/streaming-chat.md

@@ -64,6 +64,7 @@ Notes:
 - For persisted streams, backend stores only new non-assistant input history rows to avoid duplicates.
 - `additionalSystemPrompt`, when present directly or loaded from stored chat settings, is prepended to the provider request as a `system` message and is not inserted into the persisted chat transcript by this endpoint.
 - `enabledTools` limits Sybil-managed tools for this request. When omitted for a saved chat, the stored chat setting is used; otherwise all available tools are enabled by default. An empty array disables Sybil-managed tools.
+- `maxTokens` is optional. For `anthropic`, when omitted the backend requests the selected model's maximum output token limit from Anthropic's Models API and uses that as `max_tokens`; if the model limit cannot be loaded, the fallback is 128000. For other providers, omitted `maxTokens` is not sent as an explicit cap.
 - Attachments are optional and are persisted under `message.metadata.attachments` on stored user messages when `persist` is `true`.
 
 Persisted chat streams with a `chatId` are backend-owned active runs:

server/src/llm/protocols/messages-api.ts

@@ -28,6 +28,45 @@ import type { ChatMessage } from "../types.js";
 const INTERNAL_CORRECTION =
   "Internal correction: the previous assistant message claimed it would run a tool, but no tool call was made. If the task needs an available tool, call it now. Otherwise provide the final answer directly without saying you will run a tool.";
 
+const DEFAULT_ANTHROPIC_MAX_TOKENS = 128_000;
+const MODEL_MAX_TOKENS_CACHE_MS = 24 * 60 * 60 * 1000;
+
+const modelMaxTokensCache = new Map<string, { maxTokens: number; expiresAt: number }>();
+
+function readMaxTokens(value: unknown) {
+  return Number.isSafeInteger(value) && (value as number) > 0 ? (value as number) : undefined;
+}
+
+function getModelInfoMaxTokens(modelInfo: any) {
+  return readMaxTokens(modelInfo?.max_tokens) ?? readMaxTokens(modelInfo?.maxTokens);
+}
+
+async function getMessagesMaxTokens(params: ToolAwareCompletionParams) {
+  if (params.maxTokens) return params.maxTokens;
+
+  const cached = modelMaxTokensCache.get(params.model);
+  if (cached && cached.expiresAt > Date.now()) return cached.maxTokens;
+
+  try {
+    const retrieve = params.client?.models?.retrieve;
+    if (typeof retrieve === "function") {
+      const modelInfo = await retrieve.call(params.client.models, params.model);
+      const maxTokens = getModelInfoMaxTokens(modelInfo);
+      if (maxTokens) {
+        modelMaxTokensCache.set(params.model, {
+          maxTokens,
+          expiresAt: Date.now() + MODEL_MAX_TOKENS_CACHE_MS,
+        });
+        return maxTokens;
+      }
+    }
+  } catch {
+    // Fall back to the documented max for Claude Opus 4.8 and related high-output models.
+  }
+
+  return DEFAULT_ANTHROPIC_MAX_TOKENS;
+}
+
 function toTools(tools: any[]) {
   return tools
     .map((tool) => {
@@ -160,11 +199,12 @@ function mergeUsage(acc: Required<ToolAwareUsage>, usage: any) {
 
 export async function completeWithMessagesApi(params: ToolAwareCompletionParams): Promise<ToolAwareCompletionResult> {
   const enabledTools = getEnabledChatTools(params);
+  const maxTokens = await getMessagesMaxTokens(params);
   if (!enabledTools.length) {
     const response = await params.client.messages.create({
       model: params.model,
       system: buildTopLevelSystemPrompt(params.messages, params.userLocation),
-      max_tokens: params.maxTokens ?? 1024,
+      max_tokens: maxTokens,
       temperature: params.temperature,
       messages: buildBaseMessages(params),
     } as any);
@@ -192,7 +232,7 @@ export async function completeWithMessagesApi(params: ToolAwareCompletionParams)
     const response = await params.client.messages.create({
       model: params.model,
       system: buildTopLevelSystemPrompt(params.messages, params.userLocation, buildChatToolSystemPrompt(params)),
-      max_tokens: params.maxTokens ?? 1024,
+      max_tokens: maxTokens,
       temperature: params.temperature,
       messages: conversation,
       tools: toTools(enabledTools),
@@ -248,6 +288,7 @@ export async function completeWithMessagesApi(params: ToolAwareCompletionParams)
 
 export async function* streamWithMessagesApi(params: ToolAwareCompletionParams): AsyncGenerator<ToolAwareStreamingEvent> {
   const enabledTools = getEnabledChatTools(params);
+  const maxTokens = await getMessagesMaxTokens(params);
   if (!enabledTools.length) {
     const rawResponses: unknown[] = [];
     const usageAcc: Required<ToolAwareUsage> = { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
@@ -259,7 +300,7 @@ export async function* streamWithMessagesApi(params: ToolAwareCompletionParams):
     const stream = await params.client.messages.create({
       model: params.model,
       system: buildTopLevelSystemPrompt(params.messages, params.userLocation),
-      max_tokens: params.maxTokens ?? 1024,
+      max_tokens: maxTokens,
       temperature: params.temperature,
       messages: buildBaseMessages(params),
       stream: true,
@@ -315,7 +356,7 @@ export async function* streamWithMessagesApi(params: ToolAwareCompletionParams):
     const stream = await params.client.messages.create({
       model: params.model,
       system: buildTopLevelSystemPrompt(params.messages, params.userLocation, buildChatToolSystemPrompt(params)),
-      max_tokens: params.maxTokens ?? 1024,
+      max_tokens: maxTokens,
       temperature: params.temperature,
       messages: conversation,
       tools: toTools(enabledTools),

server/tests/chat-tools-streaming.test.ts

@@ -140,6 +140,94 @@ test("plain Chat Completions stream does not send Sybil-managed tools", async ()
   assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hi");
 });
 
+test("Messages API defaults max_tokens to the Anthropic model maximum", async () => {
+  let requestBody: any = null;
+  let retrievedModel: string | null = null;
+  const client = {
+    models: {
+      retrieve: async (model: string) => {
+        retrievedModel = model;
+        return { id: model, max_tokens: 128000 };
+      },
+    },
+    messages: {
+      create: async (body: any) => {
+        requestBody = body;
+        return {
+          content: [{ type: "text", text: "Done" }],
+          usage: { input_tokens: 1, output_tokens: 1 },
+        };
+      },
+    },
+  };
+
+  const result = await completeWithMessagesApi({
+    client: client as any,
+    model: "claude-max-default-test",
+    messages: [{ role: "user", content: "Say done" }],
+  });
+
+  assert.equal(retrievedModel, "claude-max-default-test");
+  assert.equal(requestBody?.max_tokens, 128000);
+  assert.equal(result.text, "Done");
+});
+
+test("Messages API preserves explicit maxTokens", async () => {
+  let requestBody: any = null;
+  let didRetrieveModel = false;
+  const client = {
+    models: {
+      retrieve: async () => {
+        didRetrieveModel = true;
+        return { max_tokens: 128000 };
+      },
+    },
+    messages: {
+      create: async (body: any) => {
+        requestBody = body;
+        return streamFrom([
+          {
+            type: "message_start",
+            message: {
+              usage: { input_tokens: 1, output_tokens: 0 },
+            },
+          },
+          {
+            type: "content_block_start",
+            index: 0,
+            content_block: { type: "text", text: "" },
+          },
+          {
+            type: "content_block_delta",
+            index: 0,
+            delta: { type: "text_delta", text: "Done" },
+          },
+          { type: "content_block_stop", index: 0 },
+          {
+            type: "message_delta",
+            delta: { stop_reason: "end_turn", stop_sequence: null },
+            usage: { output_tokens: 1 },
+          },
+          { type: "message_stop" },
+        ]);
+      },
+    },
+  };
+
+  const events = await collectEvents(
+    streamWithMessagesApi({
+      client: client as any,
+      model: "claude-explicit-max-test",
+      messages: [{ role: "user", content: "Say done" }],
+      maxTokens: 4096,
+    })
+  );
+
+  assert.equal(didRetrieveModel, false);
+  assert.equal(requestBody?.max_tokens, 4096);
+  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Done");
+});
+
 test("fetch_url sends browser-like navigation headers", async () => {
   const originalFetch = globalThis.fetch;
   const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];