import assert from "node:assert/strict"; import test from "node:test"; import { type ToolAwareStreamingEvent } from "../src/llm/chat-tools.js"; import { completeWithChatCompletionsApi, streamWithChatCompletionsApi } from "../src/llm/protocols/chat-completions-api.js"; import { completeWithMessagesApi, streamWithMessagesApi } from "../src/llm/protocols/messages-api.js"; import { streamWithResponsesApi } from "../src/llm/protocols/responses-api.js"; async function* streamFrom(events: any[]) { for (const event of events) { await Promise.resolve(); yield event; } } async function collectEvents(iterable: AsyncIterable) { const events: ToolAwareStreamingEvent[] = []; for await (const event of iterable) { events.push(event); } return events; } test("Responses API stream emits text deltas as they arrive", async () => { const outputMessage = { id: "msg_1", type: "message", role: "assistant", status: "completed", content: [{ type: "output_text", text: "Hello" }], }; const client = { responses: { create: async () => streamFrom([ { type: "response.output_item.added", item: { ...outputMessage, content: [] }, output_index: 0 }, { type: "response.output_text.delta", delta: "Hel", output_index: 0, content_index: 0 }, { type: "response.output_text.delta", delta: "lo", output_index: 0, content_index: 0 }, { type: "response.output_item.done", item: outputMessage, output_index: 0 }, { type: "response.completed", response: { status: "completed", output_text: "Hello", output: [outputMessage], usage: { input_tokens: 2, output_tokens: 1, total_tokens: 3 }, }, }, ]), }, }; const events = await collectEvents( streamWithResponsesApi({ client: client as any, model: "gpt-test", messages: [{ role: "user", content: "Say hello" }], }) ); assert.deepEqual( events.map((event) => event.type), ["delta", "delta", "done"] ); assert.deepEqual( events.filter((event) => event.type === "delta").map((event) => event.text), ["Hel", "lo"] ); assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hello"); }); test("Chat Completions API stream emits text deltas as they arrive", async () => { const client = { chat: { completions: { create: async () => streamFrom([ { choices: [{ delta: { role: "assistant" } }] }, { choices: [{ delta: { content: "Hel" } }] }, { choices: [{ delta: { content: "lo" } }] }, { choices: [{ delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 }, }, ]), }, }, }; const events = await collectEvents( streamWithChatCompletionsApi({ client: client as any, model: "grok-test", messages: [{ role: "user", content: "Say hello" }], }) ); assert.deepEqual( events.map((event) => event.type), ["delta", "delta", "done"] ); assert.deepEqual( events.filter((event) => event.type === "delta").map((event) => event.text), ["Hel", "lo"] ); assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hello"); }); test("plain Chat Completions stream does not send Sybil-managed tools", async () => { let requestBody: any = null; const client = { chat: { completions: { create: async (body: any) => { requestBody = body; return streamFrom([ { choices: [{ delta: { content: "Hi" } }] }, { choices: [{ delta: {}, finish_reason: "stop" }] }, ]); }, }, }, }; const events = await collectEvents( streamWithChatCompletionsApi({ client: client as any, model: "hermes-agent", messages: [{ role: "user", content: "Say hi" }], enabledTools: [], }) ); assert.equal(requestBody.model, "hermes-agent"); assert.equal(requestBody.stream, true); assert.equal("tools" in requestBody, false); assert.deepEqual( events.map((event) => event.type), ["delta", "done"] ); assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hi"); }); test("fetch_url sends browser-like navigation headers", async () => { const originalFetch = globalThis.fetch; const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = []; globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { fetchCalls.push({ input, init }); return new Response("CPI
Consumer price index
", { status: 200, headers: { "content-type": "text/html; charset=utf-8" }, }); }) as typeof fetch; try { let requestCount = 0; const client = { chat: { completions: { create: async () => { requestCount += 1; if (requestCount === 1) { return { choices: [ { message: { tool_calls: [ { id: "call_1", type: "function", function: { name: "fetch_url", arguments: JSON.stringify({ url: "https://www.bls.gov/news.release/pdf/cpi.pdf" }), }, }, ], }, }, ], }; } return { choices: [{ message: { content: "Fetched" } }], }; }, }, }, }; const result = await completeWithChatCompletionsApi({ client: client as any, model: "grok-test", messages: [{ role: "user", content: "Fetch CPI PDF" }], }); assert.equal(result.text, "Fetched"); assert.equal(fetchCalls.length, 1); assert.equal(String(fetchCalls[0]?.input), "https://www.bls.gov/news.release/pdf/cpi.pdf"); assert.deepEqual(fetchCalls[0]?.init?.headers, { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,application/pdf;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9", "Upgrade-Insecure-Requests": "1", "Sec-Fetch-Dest": "document", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "none", "Sec-Fetch-User": "?1", }); assert.equal(result.toolEvents[0]?.status, "completed"); } finally { globalThis.fetch = originalFetch; } }); test("Messages API executes tool_use blocks and sends tool_result follow-up", async () => { const originalFetch = globalThis.fetch; const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = []; globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { fetchCalls.push({ input, init }); return new Response("Example
Tool result body
", { status: 200, headers: { "content-type": "text/html; charset=utf-8" }, }); }) as typeof fetch; try { const requestBodies: any[] = []; const client = { messages: { create: async (body: any) => { requestBodies.push(body); if (requestBodies.length === 1) { return { content: [ { type: "tool_use", id: "toolu_1", name: "fetch_url", input: { url: "https://example.com/article" }, }, ], usage: { input_tokens: 3, output_tokens: 2 }, }; } return { content: [{ type: "text", text: "Fetched" }], usage: { input_tokens: 5, output_tokens: 1 }, }; }, }, }; const result = await completeWithMessagesApi({ client: client as any, model: "claude-test", messages: [{ role: "user", content: "Fetch the article" }], }); assert.equal(result.text, "Fetched"); assert.equal(fetchCalls.length, 1); assert.equal(String(fetchCalls[0]?.input), "https://example.com/article"); assert.equal(requestBodies.length, 2); assert.equal(requestBodies[0]?.model, "claude-test"); assert.equal(requestBodies[0]?.tool_choice?.type, "auto"); const fetchTool = requestBodies[0]?.tools?.find((tool: any) => tool.name === "fetch_url"); assert.equal(fetchTool?.input_schema?.type, "object"); assert.equal(fetchTool?.input_schema?.properties?.url?.type, "string"); const secondMessages = requestBodies[1]?.messages ?? []; assert.equal(secondMessages.at(-2)?.role, "assistant"); assert.equal(secondMessages.at(-2)?.content?.[0]?.type, "tool_use"); assert.equal(secondMessages.at(-1)?.role, "user"); const toolResult = secondMessages.at(-1)?.content?.[0]; assert.equal(toolResult?.type, "tool_result"); assert.equal(toolResult?.tool_use_id, "toolu_1"); assert.equal(toolResult?.is_error, false); assert.equal(JSON.parse(toolResult?.content ?? "{}").ok, true); assert.equal(result.toolEvents[0]?.toolCallId, "toolu_1"); assert.equal(result.toolEvents[0]?.status, "completed"); assert.equal(result.usage?.inputTokens, 8); assert.equal(result.usage?.outputTokens, 3); assert.equal(result.usage?.totalTokens, 11); } finally { globalThis.fetch = originalFetch; } }); test("Chat Completions API stream emits initiated and terminal tool call updates", async () => { let requestCount = 0; const client = { chat: { completions: { create: async () => { requestCount += 1; if (requestCount === 1) { return streamFrom([ { choices: [ { delta: { tool_calls: [ { index: 0, id: "call_1", function: { name: "unknown_tool", arguments: "{\"query\":\"current weather\"}", }, }, ], }, finish_reason: "tool_calls", }, ], }, ]); } return streamFrom([ { choices: [{ delta: { content: "Done" } }] }, { choices: [{ delta: {}, finish_reason: "stop" }] }, ]); }, }, }, }; const events = await collectEvents( streamWithChatCompletionsApi({ client: client as any, model: "grok-test", messages: [{ role: "user", content: "Use a tool" }], }) ); assert.deepEqual( events.map((event) => event.type), ["tool_call", "tool_call", "delta", "done"] ); const toolEvents = events.flatMap((event) => (event.type === "tool_call" ? [event.event] : [])); assert.equal(toolEvents[0]?.toolCallId, "call_1"); assert.equal(toolEvents[0]?.status, "initiated"); assert.equal(toolEvents[0]?.completedAt, undefined); assert.equal(toolEvents[0]?.durationMs, undefined); assert.equal(toolEvents[1]?.toolCallId, "call_1"); assert.equal(toolEvents[1]?.status, "failed"); assert.match(toolEvents[1]?.error ?? "", /Unknown tool: unknown_tool/); assert.equal(typeof toolEvents[1]?.completedAt, "string"); assert.equal(typeof toolEvents[1]?.durationMs, "number"); assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Done"); }); test("Messages API stream emits initiated and terminal tool call updates", async () => { let requestCount = 0; const requestBodies: any[] = []; const client = { messages: { create: async (body: any) => { requestCount += 1; requestBodies.push(body); if (requestCount === 1) { return streamFrom([ { type: "message_start", message: { usage: { input_tokens: 3, output_tokens: 0 }, }, }, { type: "content_block_start", index: 0, content_block: { type: "text", text: "" }, }, { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "I'll check that." }, }, { type: "content_block_stop", index: 0 }, { type: "content_block_start", index: 1, content_block: { type: "tool_use", id: "toolu_1", name: "unknown_tool", input: {}, }, }, { type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: "{\"query\":\"current weather\"}" }, }, { type: "content_block_stop", index: 1 }, { type: "message_delta", delta: { stop_reason: "tool_use", stop_sequence: null }, usage: { output_tokens: 2 }, }, { type: "message_stop" }, ]); } return streamFrom([ { type: "message_start", message: { usage: { input_tokens: 4, output_tokens: 0 }, }, }, { type: "content_block_start", index: 0, content_block: { type: "text", text: "" }, }, { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Done" }, }, { type: "content_block_stop", index: 0 }, { type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 1 }, }, { type: "message_stop" }, ]); }, }, }; const events = await collectEvents( streamWithMessagesApi({ client: client as any, model: "claude-test", messages: [{ role: "user", content: "Use a tool" }], }) ); assert.deepEqual( events.map((event) => event.type), ["tool_call", "tool_call", "delta", "done"] ); assert.equal(requestBodies[0]?.stream, true); assert.equal(requestBodies[0]?.tools?.some((tool: any) => tool.name === "fetch_url"), true); const secondMessages = requestBodies[1]?.messages ?? []; assert.equal(secondMessages.at(-2)?.role, "assistant"); assert.equal(secondMessages.at(-2)?.content?.[0]?.type, "text"); assert.equal(secondMessages.at(-2)?.content?.[0]?.text, "I'll check that."); assert.equal(secondMessages.at(-2)?.content?.[1]?.type, "tool_use"); assert.deepEqual(secondMessages.at(-2)?.content?.[1]?.input, { query: "current weather" }); const toolResult = secondMessages.at(-1)?.content?.[0]; assert.equal(toolResult?.type, "tool_result"); assert.equal(toolResult?.tool_use_id, "toolu_1"); assert.equal(toolResult?.is_error, true); assert.match(JSON.parse(toolResult?.content ?? "{}").error ?? "", /Unknown tool: unknown_tool/); const toolEvents = events.flatMap((event) => (event.type === "tool_call" ? [event.event] : [])); assert.equal(toolEvents[0]?.toolCallId, "toolu_1"); assert.equal(toolEvents[0]?.status, "initiated"); assert.equal(toolEvents[1]?.toolCallId, "toolu_1"); assert.equal(toolEvents[1]?.status, "failed"); assert.match(toolEvents[1]?.error ?? "", /Unknown tool: unknown_tool/); assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Done"); assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.usage?.inputTokens : null, 7); assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.usage?.outputTokens : null, 3); });