474 lines
16 KiB
TypeScript
474 lines
16 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
import { type ToolAwareStreamingEvent } from "../src/llm/chat-tools.js";
|
|
import { completeWithChatCompletionsApi, streamWithChatCompletionsApi } from "../src/llm/protocols/chat-completions-api.js";
|
|
import { completeWithMessagesApi, streamWithMessagesApi } from "../src/llm/protocols/messages-api.js";
|
|
import { streamWithResponsesApi } from "../src/llm/protocols/responses-api.js";
|
|
|
|
async function* streamFrom(events: any[]) {
|
|
for (const event of events) {
|
|
await Promise.resolve();
|
|
yield event;
|
|
}
|
|
}
|
|
|
|
async function collectEvents(iterable: AsyncIterable<ToolAwareStreamingEvent>) {
|
|
const events: ToolAwareStreamingEvent[] = [];
|
|
for await (const event of iterable) {
|
|
events.push(event);
|
|
}
|
|
return events;
|
|
}
|
|
|
|
test("Responses API stream emits text deltas as they arrive", async () => {
|
|
const outputMessage = {
|
|
id: "msg_1",
|
|
type: "message",
|
|
role: "assistant",
|
|
status: "completed",
|
|
content: [{ type: "output_text", text: "Hello" }],
|
|
};
|
|
const client = {
|
|
responses: {
|
|
create: async () =>
|
|
streamFrom([
|
|
{ type: "response.output_item.added", item: { ...outputMessage, content: [] }, output_index: 0 },
|
|
{ type: "response.output_text.delta", delta: "Hel", output_index: 0, content_index: 0 },
|
|
{ type: "response.output_text.delta", delta: "lo", output_index: 0, content_index: 0 },
|
|
{ type: "response.output_item.done", item: outputMessage, output_index: 0 },
|
|
{
|
|
type: "response.completed",
|
|
response: {
|
|
status: "completed",
|
|
output_text: "Hello",
|
|
output: [outputMessage],
|
|
usage: { input_tokens: 2, output_tokens: 1, total_tokens: 3 },
|
|
},
|
|
},
|
|
]),
|
|
},
|
|
};
|
|
|
|
const events = await collectEvents(
|
|
streamWithResponsesApi({
|
|
client: client as any,
|
|
model: "gpt-test",
|
|
messages: [{ role: "user", content: "Say hello" }],
|
|
})
|
|
);
|
|
|
|
assert.deepEqual(
|
|
events.map((event) => event.type),
|
|
["delta", "delta", "done"]
|
|
);
|
|
assert.deepEqual(
|
|
events.filter((event) => event.type === "delta").map((event) => event.text),
|
|
["Hel", "lo"]
|
|
);
|
|
assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hello");
|
|
});
|
|
|
|
test("Chat Completions API stream emits text deltas as they arrive", async () => {
|
|
const client = {
|
|
chat: {
|
|
completions: {
|
|
create: async () =>
|
|
streamFrom([
|
|
{ choices: [{ delta: { role: "assistant" } }] },
|
|
{ choices: [{ delta: { content: "Hel" } }] },
|
|
{ choices: [{ delta: { content: "lo" } }] },
|
|
{
|
|
choices: [{ delta: {}, finish_reason: "stop" }],
|
|
usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },
|
|
},
|
|
]),
|
|
},
|
|
},
|
|
};
|
|
|
|
const events = await collectEvents(
|
|
streamWithChatCompletionsApi({
|
|
client: client as any,
|
|
model: "grok-test",
|
|
messages: [{ role: "user", content: "Say hello" }],
|
|
})
|
|
);
|
|
|
|
assert.deepEqual(
|
|
events.map((event) => event.type),
|
|
["delta", "delta", "done"]
|
|
);
|
|
assert.deepEqual(
|
|
events.filter((event) => event.type === "delta").map((event) => event.text),
|
|
["Hel", "lo"]
|
|
);
|
|
assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hello");
|
|
});
|
|
|
|
test("plain Chat Completions stream does not send Sybil-managed tools", async () => {
|
|
let requestBody: any = null;
|
|
const client = {
|
|
chat: {
|
|
completions: {
|
|
create: async (body: any) => {
|
|
requestBody = body;
|
|
return streamFrom([
|
|
{ choices: [{ delta: { content: "Hi" } }] },
|
|
{ choices: [{ delta: {}, finish_reason: "stop" }] },
|
|
]);
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
const events = await collectEvents(
|
|
streamWithChatCompletionsApi({
|
|
client: client as any,
|
|
model: "hermes-agent",
|
|
messages: [{ role: "user", content: "Say hi" }],
|
|
enabledTools: [],
|
|
})
|
|
);
|
|
|
|
assert.equal(requestBody.model, "hermes-agent");
|
|
assert.equal(requestBody.stream, true);
|
|
assert.equal("tools" in requestBody, false);
|
|
assert.deepEqual(
|
|
events.map((event) => event.type),
|
|
["delta", "done"]
|
|
);
|
|
assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hi");
|
|
});
|
|
|
|
test("fetch_url sends browser-like navigation headers", async () => {
|
|
const originalFetch = globalThis.fetch;
|
|
const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
|
|
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
|
fetchCalls.push({ input, init });
|
|
return new Response("<!doctype html><title>CPI</title><main>Consumer price index</main>", {
|
|
status: 200,
|
|
headers: { "content-type": "text/html; charset=utf-8" },
|
|
});
|
|
}) as typeof fetch;
|
|
|
|
try {
|
|
let requestCount = 0;
|
|
const client = {
|
|
chat: {
|
|
completions: {
|
|
create: async () => {
|
|
requestCount += 1;
|
|
if (requestCount === 1) {
|
|
return {
|
|
choices: [
|
|
{
|
|
message: {
|
|
tool_calls: [
|
|
{
|
|
id: "call_1",
|
|
type: "function",
|
|
function: {
|
|
name: "fetch_url",
|
|
arguments: JSON.stringify({ url: "https://www.bls.gov/news.release/pdf/cpi.pdf" }),
|
|
},
|
|
},
|
|
],
|
|
},
|
|
},
|
|
],
|
|
};
|
|
}
|
|
|
|
return {
|
|
choices: [{ message: { content: "Fetched" } }],
|
|
};
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
const result = await completeWithChatCompletionsApi({
|
|
client: client as any,
|
|
model: "grok-test",
|
|
messages: [{ role: "user", content: "Fetch CPI PDF" }],
|
|
});
|
|
|
|
assert.equal(result.text, "Fetched");
|
|
assert.equal(fetchCalls.length, 1);
|
|
assert.equal(String(fetchCalls[0]?.input), "https://www.bls.gov/news.release/pdf/cpi.pdf");
|
|
assert.deepEqual(fetchCalls[0]?.init?.headers, {
|
|
"User-Agent":
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
|
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,application/pdf;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
"Upgrade-Insecure-Requests": "1",
|
|
"Sec-Fetch-Dest": "document",
|
|
"Sec-Fetch-Mode": "navigate",
|
|
"Sec-Fetch-Site": "none",
|
|
"Sec-Fetch-User": "?1",
|
|
});
|
|
assert.equal(result.toolEvents[0]?.status, "completed");
|
|
} finally {
|
|
globalThis.fetch = originalFetch;
|
|
}
|
|
});
|
|
|
|
test("Messages API executes tool_use blocks and sends tool_result follow-up", async () => {
|
|
const originalFetch = globalThis.fetch;
|
|
const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
|
|
globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
|
fetchCalls.push({ input, init });
|
|
return new Response("<!doctype html><title>Example</title><main>Tool result body</main>", {
|
|
status: 200,
|
|
headers: { "content-type": "text/html; charset=utf-8" },
|
|
});
|
|
}) as typeof fetch;
|
|
|
|
try {
|
|
const requestBodies: any[] = [];
|
|
const client = {
|
|
messages: {
|
|
create: async (body: any) => {
|
|
requestBodies.push(body);
|
|
if (requestBodies.length === 1) {
|
|
return {
|
|
content: [
|
|
{
|
|
type: "tool_use",
|
|
id: "toolu_1",
|
|
name: "fetch_url",
|
|
input: { url: "https://example.com/article" },
|
|
},
|
|
],
|
|
usage: { input_tokens: 3, output_tokens: 2 },
|
|
};
|
|
}
|
|
|
|
return {
|
|
content: [{ type: "text", text: "Fetched" }],
|
|
usage: { input_tokens: 5, output_tokens: 1 },
|
|
};
|
|
},
|
|
},
|
|
};
|
|
|
|
const result = await completeWithMessagesApi({
|
|
client: client as any,
|
|
model: "claude-test",
|
|
messages: [{ role: "user", content: "Fetch the article" }],
|
|
});
|
|
|
|
assert.equal(result.text, "Fetched");
|
|
assert.equal(fetchCalls.length, 1);
|
|
assert.equal(String(fetchCalls[0]?.input), "https://example.com/article");
|
|
assert.equal(requestBodies.length, 2);
|
|
assert.equal(requestBodies[0]?.model, "claude-test");
|
|
assert.equal(requestBodies[0]?.tool_choice?.type, "auto");
|
|
const fetchTool = requestBodies[0]?.tools?.find((tool: any) => tool.name === "fetch_url");
|
|
assert.equal(fetchTool?.input_schema?.type, "object");
|
|
assert.equal(fetchTool?.input_schema?.properties?.url?.type, "string");
|
|
|
|
const secondMessages = requestBodies[1]?.messages ?? [];
|
|
assert.equal(secondMessages.at(-2)?.role, "assistant");
|
|
assert.equal(secondMessages.at(-2)?.content?.[0]?.type, "tool_use");
|
|
assert.equal(secondMessages.at(-1)?.role, "user");
|
|
const toolResult = secondMessages.at(-1)?.content?.[0];
|
|
assert.equal(toolResult?.type, "tool_result");
|
|
assert.equal(toolResult?.tool_use_id, "toolu_1");
|
|
assert.equal(toolResult?.is_error, false);
|
|
assert.equal(JSON.parse(toolResult?.content ?? "{}").ok, true);
|
|
assert.equal(result.toolEvents[0]?.toolCallId, "toolu_1");
|
|
assert.equal(result.toolEvents[0]?.status, "completed");
|
|
assert.equal(result.usage?.inputTokens, 8);
|
|
assert.equal(result.usage?.outputTokens, 3);
|
|
assert.equal(result.usage?.totalTokens, 11);
|
|
} finally {
|
|
globalThis.fetch = originalFetch;
|
|
}
|
|
});
|
|
|
|
test("Chat Completions API stream emits initiated and terminal tool call updates", async () => {
|
|
let requestCount = 0;
|
|
const client = {
|
|
chat: {
|
|
completions: {
|
|
create: async () => {
|
|
requestCount += 1;
|
|
if (requestCount === 1) {
|
|
return streamFrom([
|
|
{
|
|
choices: [
|
|
{
|
|
delta: {
|
|
tool_calls: [
|
|
{
|
|
index: 0,
|
|
id: "call_1",
|
|
function: {
|
|
name: "unknown_tool",
|
|
arguments: "{\"query\":\"current weather\"}",
|
|
},
|
|
},
|
|
],
|
|
},
|
|
finish_reason: "tool_calls",
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
}
|
|
|
|
return streamFrom([
|
|
{ choices: [{ delta: { content: "Done" } }] },
|
|
{ choices: [{ delta: {}, finish_reason: "stop" }] },
|
|
]);
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
const events = await collectEvents(
|
|
streamWithChatCompletionsApi({
|
|
client: client as any,
|
|
model: "grok-test",
|
|
messages: [{ role: "user", content: "Use a tool" }],
|
|
})
|
|
);
|
|
|
|
assert.deepEqual(
|
|
events.map((event) => event.type),
|
|
["tool_call", "tool_call", "delta", "done"]
|
|
);
|
|
|
|
const toolEvents = events.flatMap((event) => (event.type === "tool_call" ? [event.event] : []));
|
|
assert.equal(toolEvents[0]?.toolCallId, "call_1");
|
|
assert.equal(toolEvents[0]?.status, "initiated");
|
|
assert.equal(toolEvents[0]?.completedAt, undefined);
|
|
assert.equal(toolEvents[0]?.durationMs, undefined);
|
|
assert.equal(toolEvents[1]?.toolCallId, "call_1");
|
|
assert.equal(toolEvents[1]?.status, "failed");
|
|
assert.match(toolEvents[1]?.error ?? "", /Unknown tool: unknown_tool/);
|
|
assert.equal(typeof toolEvents[1]?.completedAt, "string");
|
|
assert.equal(typeof toolEvents[1]?.durationMs, "number");
|
|
assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Done");
|
|
});
|
|
|
|
test("Messages API stream emits initiated and terminal tool call updates", async () => {
|
|
let requestCount = 0;
|
|
const requestBodies: any[] = [];
|
|
const client = {
|
|
messages: {
|
|
create: async (body: any) => {
|
|
requestCount += 1;
|
|
requestBodies.push(body);
|
|
if (requestCount === 1) {
|
|
return streamFrom([
|
|
{
|
|
type: "message_start",
|
|
message: {
|
|
usage: { input_tokens: 3, output_tokens: 0 },
|
|
},
|
|
},
|
|
{
|
|
type: "content_block_start",
|
|
index: 0,
|
|
content_block: { type: "text", text: "" },
|
|
},
|
|
{
|
|
type: "content_block_delta",
|
|
index: 0,
|
|
delta: { type: "text_delta", text: "I'll check that." },
|
|
},
|
|
{ type: "content_block_stop", index: 0 },
|
|
{
|
|
type: "content_block_start",
|
|
index: 1,
|
|
content_block: {
|
|
type: "tool_use",
|
|
id: "toolu_1",
|
|
name: "unknown_tool",
|
|
input: {},
|
|
},
|
|
},
|
|
{
|
|
type: "content_block_delta",
|
|
index: 1,
|
|
delta: { type: "input_json_delta", partial_json: "{\"query\":\"current weather\"}" },
|
|
},
|
|
{ type: "content_block_stop", index: 1 },
|
|
{
|
|
type: "message_delta",
|
|
delta: { stop_reason: "tool_use", stop_sequence: null },
|
|
usage: { output_tokens: 2 },
|
|
},
|
|
{ type: "message_stop" },
|
|
]);
|
|
}
|
|
|
|
return streamFrom([
|
|
{
|
|
type: "message_start",
|
|
message: {
|
|
usage: { input_tokens: 4, output_tokens: 0 },
|
|
},
|
|
},
|
|
{
|
|
type: "content_block_start",
|
|
index: 0,
|
|
content_block: { type: "text", text: "" },
|
|
},
|
|
{
|
|
type: "content_block_delta",
|
|
index: 0,
|
|
delta: { type: "text_delta", text: "Done" },
|
|
},
|
|
{ type: "content_block_stop", index: 0 },
|
|
{
|
|
type: "message_delta",
|
|
delta: { stop_reason: "end_turn", stop_sequence: null },
|
|
usage: { output_tokens: 1 },
|
|
},
|
|
{ type: "message_stop" },
|
|
]);
|
|
},
|
|
},
|
|
};
|
|
|
|
const events = await collectEvents(
|
|
streamWithMessagesApi({
|
|
client: client as any,
|
|
model: "claude-test",
|
|
messages: [{ role: "user", content: "Use a tool" }],
|
|
})
|
|
);
|
|
|
|
assert.deepEqual(
|
|
events.map((event) => event.type),
|
|
["tool_call", "tool_call", "delta", "done"]
|
|
);
|
|
assert.equal(requestBodies[0]?.stream, true);
|
|
assert.equal(requestBodies[0]?.tools?.some((tool: any) => tool.name === "fetch_url"), true);
|
|
|
|
const secondMessages = requestBodies[1]?.messages ?? [];
|
|
assert.equal(secondMessages.at(-2)?.role, "assistant");
|
|
assert.equal(secondMessages.at(-2)?.content?.[0]?.type, "text");
|
|
assert.equal(secondMessages.at(-2)?.content?.[0]?.text, "I'll check that.");
|
|
assert.equal(secondMessages.at(-2)?.content?.[1]?.type, "tool_use");
|
|
assert.deepEqual(secondMessages.at(-2)?.content?.[1]?.input, { query: "current weather" });
|
|
const toolResult = secondMessages.at(-1)?.content?.[0];
|
|
assert.equal(toolResult?.type, "tool_result");
|
|
assert.equal(toolResult?.tool_use_id, "toolu_1");
|
|
assert.equal(toolResult?.is_error, true);
|
|
assert.match(JSON.parse(toolResult?.content ?? "{}").error ?? "", /Unknown tool: unknown_tool/);
|
|
|
|
const toolEvents = events.flatMap((event) => (event.type === "tool_call" ? [event.event] : []));
|
|
assert.equal(toolEvents[0]?.toolCallId, "toolu_1");
|
|
assert.equal(toolEvents[0]?.status, "initiated");
|
|
assert.equal(toolEvents[1]?.toolCallId, "toolu_1");
|
|
assert.equal(toolEvents[1]?.status, "failed");
|
|
assert.match(toolEvents[1]?.error ?? "", /Unknown tool: unknown_tool/);
|
|
assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Done");
|
|
assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.usage?.inputTokens : null, 7);
|
|
assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.usage?.outputTokens : null, 3);
|
|
});
|