Sybil-2/server/tests/chat-tools-streaming.test.ts

import assert from "node:assert/strict";
import test from "node:test";
import { type ToolAwareStreamingEvent } from "../src/llm/chat-tools.js";
import { completeWithChatCompletionsApi, streamWithChatCompletionsApi } from "../src/llm/protocols/chat-completions-api.js";
import { completeWithMessagesApi, streamWithMessagesApi } from "../src/llm/protocols/messages-api.js";
import { streamWithResponsesApi } from "../src/llm/protocols/responses-api.js";

async function* streamFrom(events: any[]) {
  for (const event of events) {
    await Promise.resolve();
    yield event;
  }
}

async function collectEvents(iterable: AsyncIterable<ToolAwareStreamingEvent>) {
  const events: ToolAwareStreamingEvent[] = [];
  for await (const event of iterable) {
    events.push(event);
  }
  return events;
}

test("Responses API stream emits text deltas as they arrive", async () => {
  const outputMessage = {
    id: "msg_1",
    type: "message",
    role: "assistant",
    status: "completed",
    content: [{ type: "output_text", text: "Hello" }],
  };
  const client = {
    responses: {
      create: async () =>
        streamFrom([
          { type: "response.output_item.added", item: { ...outputMessage, content: [] }, output_index: 0 },
          { type: "response.output_text.delta", delta: "Hel", output_index: 0, content_index: 0 },
          { type: "response.output_text.delta", delta: "lo", output_index: 0, content_index: 0 },
          { type: "response.output_item.done", item: outputMessage, output_index: 0 },
          {
            type: "response.completed",
            response: {
              status: "completed",
              output_text: "Hello",
              output: [outputMessage],
              usage: { input_tokens: 2, output_tokens: 1, total_tokens: 3 },
            },
          },
        ]),
    },
  };

  const events = await collectEvents(
    streamWithResponsesApi({
      client: client as any,
      model: "gpt-test",
      messages: [{ role: "user", content: "Say hello" }],
    })
  );

  assert.deepEqual(
    events.map((event) => event.type),
    ["delta", "delta", "done"]
  );
  assert.deepEqual(
    events.filter((event) => event.type === "delta").map((event) => event.text),
    ["Hel", "lo"]
  );
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hello");
});

test("Chat Completions API stream emits text deltas as they arrive", async () => {
  const client = {
    chat: {
      completions: {
        create: async () =>
          streamFrom([
            { choices: [{ delta: { role: "assistant" } }] },
            { choices: [{ delta: { content: "Hel" } }] },
            { choices: [{ delta: { content: "lo" } }] },
            {
              choices: [{ delta: {}, finish_reason: "stop" }],
              usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },
            },
          ]),
      },
    },
  };

  const events = await collectEvents(
    streamWithChatCompletionsApi({
      client: client as any,
      model: "grok-test",
      messages: [{ role: "user", content: "Say hello" }],
    })
  );

  assert.deepEqual(
    events.map((event) => event.type),
    ["delta", "delta", "done"]
  );
  assert.deepEqual(
    events.filter((event) => event.type === "delta").map((event) => event.text),
    ["Hel", "lo"]
  );
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hello");
});

test("plain Chat Completions stream does not send Sybil-managed tools", async () => {
  let requestBody: any = null;
  const client = {
    chat: {
      completions: {
        create: async (body: any) => {
          requestBody = body;
          return streamFrom([
            { choices: [{ delta: { content: "Hi" } }] },
            { choices: [{ delta: {}, finish_reason: "stop" }] },
          ]);
        },
      },
    },
  };

  const events = await collectEvents(
    streamWithChatCompletionsApi({
      client: client as any,
      model: "hermes-agent",
      messages: [{ role: "user", content: "Say hi" }],
      enabledTools: [],
    })
  );

  assert.equal(requestBody.model, "hermes-agent");
  assert.equal(requestBody.stream, true);
  assert.equal("tools" in requestBody, false);
  assert.deepEqual(
    events.map((event) => event.type),
    ["delta", "done"]
  );
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hi");
});

test("fetch_url sends browser-like navigation headers", async () => {
  const originalFetch = globalThis.fetch;
  const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
  globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
    fetchCalls.push({ input, init });
    return new Response("<!doctype html><title>CPI</title><main>Consumer price index</main>", {
      status: 200,
      headers: { "content-type": "text/html; charset=utf-8" },
    });
  }) as typeof fetch;

  try {
    let requestCount = 0;
    const client = {
      chat: {
        completions: {
          create: async () => {
            requestCount += 1;
            if (requestCount === 1) {
              return {
                choices: [
                  {
                    message: {
                      tool_calls: [
                        {
                          id: "call_1",
                          type: "function",
                          function: {
                            name: "fetch_url",
                            arguments: JSON.stringify({ url: "https://www.bls.gov/news.release/pdf/cpi.pdf" }),
                          },
                        },
                      ],
                    },
                  },
                ],
              };
            }

            return {
              choices: [{ message: { content: "Fetched" } }],
            };
          },
        },
      },
    };

    const result = await completeWithChatCompletionsApi({
      client: client as any,
      model: "grok-test",
      messages: [{ role: "user", content: "Fetch CPI PDF" }],
    });

    assert.equal(result.text, "Fetched");
    assert.equal(fetchCalls.length, 1);
    assert.equal(String(fetchCalls[0]?.input), "https://www.bls.gov/news.release/pdf/cpi.pdf");
    assert.deepEqual(fetchCalls[0]?.init?.headers, {
      "User-Agent":
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,application/pdf;q=0.9,*/*;q=0.8",
      "Accept-Language": "en-US,en;q=0.9",
      "Upgrade-Insecure-Requests": "1",
      "Sec-Fetch-Dest": "document",
      "Sec-Fetch-Mode": "navigate",
      "Sec-Fetch-Site": "none",
      "Sec-Fetch-User": "?1",
    });
    assert.equal(result.toolEvents[0]?.status, "completed");
  } finally {
    globalThis.fetch = originalFetch;
  }
});

test("Messages API executes tool_use blocks and sends tool_result follow-up", async () => {
  const originalFetch = globalThis.fetch;
  const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
  globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
    fetchCalls.push({ input, init });
    return new Response("<!doctype html><title>Example</title><main>Tool result body</main>", {
      status: 200,
      headers: { "content-type": "text/html; charset=utf-8" },
    });
  }) as typeof fetch;

  try {
    const requestBodies: any[] = [];
    const client = {
      messages: {
        create: async (body: any) => {
          requestBodies.push(body);
          if (requestBodies.length === 1) {
            return {
              content: [
                {
                  type: "tool_use",
                  id: "toolu_1",
                  name: "fetch_url",
                  input: { url: "https://example.com/article" },
                },
              ],
              usage: { input_tokens: 3, output_tokens: 2 },
            };
          }

          return {
            content: [{ type: "text", text: "Fetched" }],
            usage: { input_tokens: 5, output_tokens: 1 },
          };
        },
      },
    };

    const result = await completeWithMessagesApi({
      client: client as any,
      model: "claude-test",
      messages: [{ role: "user", content: "Fetch the article" }],
    });

    assert.equal(result.text, "Fetched");
    assert.equal(fetchCalls.length, 1);
    assert.equal(String(fetchCalls[0]?.input), "https://example.com/article");
    assert.equal(requestBodies.length, 2);
    assert.equal(requestBodies[0]?.model, "claude-test");
    assert.equal(requestBodies[0]?.tool_choice?.type, "auto");
    const fetchTool = requestBodies[0]?.tools?.find((tool: any) => tool.name === "fetch_url");
    assert.equal(fetchTool?.input_schema?.type, "object");
    assert.equal(fetchTool?.input_schema?.properties?.url?.type, "string");

    const secondMessages = requestBodies[1]?.messages ?? [];
    assert.equal(secondMessages.at(-2)?.role, "assistant");
    assert.equal(secondMessages.at(-2)?.content?.[0]?.type, "tool_use");
    assert.equal(secondMessages.at(-1)?.role, "user");
    const toolResult = secondMessages.at(-1)?.content?.[0];
    assert.equal(toolResult?.type, "tool_result");
    assert.equal(toolResult?.tool_use_id, "toolu_1");
    assert.equal(toolResult?.is_error, false);
    assert.equal(JSON.parse(toolResult?.content ?? "{}").ok, true);
    assert.equal(result.toolEvents[0]?.toolCallId, "toolu_1");
    assert.equal(result.toolEvents[0]?.status, "completed");
    assert.equal(result.usage?.inputTokens, 8);
    assert.equal(result.usage?.outputTokens, 3);
    assert.equal(result.usage?.totalTokens, 11);
  } finally {
    globalThis.fetch = originalFetch;
  }
});

test("Chat Completions API stream emits initiated and terminal tool call updates", async () => {
  let requestCount = 0;
  const client = {
    chat: {
      completions: {
        create: async () => {
          requestCount += 1;
          if (requestCount === 1) {
            return streamFrom([
              {
                choices: [
                  {
                    delta: {
                      tool_calls: [
                        {
                          index: 0,
                          id: "call_1",
                          function: {
                            name: "unknown_tool",
                            arguments: "{\"query\":\"current weather\"}",
                          },
                        },
                      ],
                    },
                    finish_reason: "tool_calls",
                  },
                ],
              },
            ]);
          }

          return streamFrom([
            { choices: [{ delta: { content: "Done" } }] },
            { choices: [{ delta: {}, finish_reason: "stop" }] },
          ]);
        },
      },
    },
  };

  const events = await collectEvents(
    streamWithChatCompletionsApi({
      client: client as any,
      model: "grok-test",
      messages: [{ role: "user", content: "Use a tool" }],
    })
  );

  assert.deepEqual(
    events.map((event) => event.type),
    ["tool_call", "tool_call", "delta", "done"]
  );

  const toolEvents = events.flatMap((event) => (event.type === "tool_call" ? [event.event] : []));
  assert.equal(toolEvents[0]?.toolCallId, "call_1");
  assert.equal(toolEvents[0]?.status, "initiated");
  assert.equal(toolEvents[0]?.completedAt, undefined);
  assert.equal(toolEvents[0]?.durationMs, undefined);
  assert.equal(toolEvents[1]?.toolCallId, "call_1");
  assert.equal(toolEvents[1]?.status, "failed");
  assert.match(toolEvents[1]?.error ?? "", /Unknown tool: unknown_tool/);
  assert.equal(typeof toolEvents[1]?.completedAt, "string");
  assert.equal(typeof toolEvents[1]?.durationMs, "number");
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Done");
});

test("Messages API stream emits initiated and terminal tool call updates", async () => {
  let requestCount = 0;
  const requestBodies: any[] = [];
  const client = {
    messages: {
      create: async (body: any) => {
        requestCount += 1;
        requestBodies.push(body);
        if (requestCount === 1) {
          return streamFrom([
            {
              type: "message_start",
              message: {
                usage: { input_tokens: 3, output_tokens: 0 },
              },
            },
            {
              type: "content_block_start",
              index: 0,
              content_block: { type: "text", text: "" },
            },
            {
              type: "content_block_delta",
              index: 0,
              delta: { type: "text_delta", text: "I'll check that." },
            },
            { type: "content_block_stop", index: 0 },
            {
              type: "content_block_start",
              index: 1,
              content_block: {
                type: "tool_use",
                id: "toolu_1",
                name: "unknown_tool",
                input: {},
              },
            },
            {
              type: "content_block_delta",
              index: 1,
              delta: { type: "input_json_delta", partial_json: "{\"query\":\"current weather\"}" },
            },
            { type: "content_block_stop", index: 1 },
            {
              type: "message_delta",
              delta: { stop_reason: "tool_use", stop_sequence: null },
              usage: { output_tokens: 2 },
            },
            { type: "message_stop" },
          ]);
        }

        return streamFrom([
          {
            type: "message_start",
            message: {
              usage: { input_tokens: 4, output_tokens: 0 },
            },
          },
          {
            type: "content_block_start",
            index: 0,
            content_block: { type: "text", text: "" },
          },
          {
            type: "content_block_delta",
            index: 0,
            delta: { type: "text_delta", text: "Done" },
          },
          { type: "content_block_stop", index: 0 },
          {
            type: "message_delta",
            delta: { stop_reason: "end_turn", stop_sequence: null },
            usage: { output_tokens: 1 },
          },
          { type: "message_stop" },
        ]);
      },
    },
  };

  const events = await collectEvents(
    streamWithMessagesApi({
      client: client as any,
      model: "claude-test",
      messages: [{ role: "user", content: "Use a tool" }],
    })
  );

  assert.deepEqual(
    events.map((event) => event.type),
    ["tool_call", "tool_call", "delta", "done"]
  );
  assert.equal(requestBodies[0]?.stream, true);
  assert.equal(requestBodies[0]?.tools?.some((tool: any) => tool.name === "fetch_url"), true);

  const secondMessages = requestBodies[1]?.messages ?? [];
  assert.equal(secondMessages.at(-2)?.role, "assistant");
  assert.equal(secondMessages.at(-2)?.content?.[0]?.type, "text");
  assert.equal(secondMessages.at(-2)?.content?.[0]?.text, "I'll check that.");
  assert.equal(secondMessages.at(-2)?.content?.[1]?.type, "tool_use");
  assert.deepEqual(secondMessages.at(-2)?.content?.[1]?.input, { query: "current weather" });
  const toolResult = secondMessages.at(-1)?.content?.[0];
  assert.equal(toolResult?.type, "tool_result");
  assert.equal(toolResult?.tool_use_id, "toolu_1");
  assert.equal(toolResult?.is_error, true);
  assert.match(JSON.parse(toolResult?.content ?? "{}").error ?? "", /Unknown tool: unknown_tool/);

  const toolEvents = events.flatMap((event) => (event.type === "tool_call" ? [event.event] : []));
  assert.equal(toolEvents[0]?.toolCallId, "toolu_1");
  assert.equal(toolEvents[0]?.status, "initiated");
  assert.equal(toolEvents[1]?.toolCallId, "toolu_1");
  assert.equal(toolEvents[1]?.status, "failed");
  assert.match(toolEvents[1]?.error ?? "", /Unknown tool: unknown_tool/);
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Done");
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.usage?.inputTokens : null, 7);
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.usage?.outputTokens : null, 3);
});