Sybil-2/server/tests/chat-tools-streaming.test.ts

import assert from "node:assert/strict";
import test from "node:test";
import {
  runPlainChatCompletionsStream,
  runToolAwareChatCompletions,
  runToolAwareChatCompletionsStream,
  runToolAwareOpenAIChatStream,
  type ToolAwareStreamingEvent,
} from "../src/llm/chat-tools.js";

async function* streamFrom(events: any[]) {
  for (const event of events) {
    await Promise.resolve();
    yield event;
  }
}

async function collectEvents(iterable: AsyncIterable<ToolAwareStreamingEvent>) {
  const events: ToolAwareStreamingEvent[] = [];
  for await (const event of iterable) {
    events.push(event);
  }
  return events;
}

test("OpenAI Responses stream emits text deltas as they arrive", async () => {
  const outputMessage = {
    id: "msg_1",
    type: "message",
    role: "assistant",
    status: "completed",
    content: [{ type: "output_text", text: "Hello" }],
  };
  const client = {
    responses: {
      create: async () =>
        streamFrom([
          { type: "response.output_item.added", item: { ...outputMessage, content: [] }, output_index: 0 },
          { type: "response.output_text.delta", delta: "Hel", output_index: 0, content_index: 0 },
          { type: "response.output_text.delta", delta: "lo", output_index: 0, content_index: 0 },
          { type: "response.output_item.done", item: outputMessage, output_index: 0 },
          {
            type: "response.completed",
            response: {
              status: "completed",
              output_text: "Hello",
              output: [outputMessage],
              usage: { input_tokens: 2, output_tokens: 1, total_tokens: 3 },
            },
          },
        ]),
    },
  };

  const events = await collectEvents(
    runToolAwareOpenAIChatStream({
      client: client as any,
      model: "gpt-test",
      messages: [{ role: "user", content: "Say hello" }],
    })
  );

  assert.deepEqual(
    events.map((event) => event.type),
    ["delta", "delta", "done"]
  );
  assert.deepEqual(
    events.filter((event) => event.type === "delta").map((event) => event.text),
    ["Hel", "lo"]
  );
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hello");
});

test("OpenAI-compatible Chat Completions stream emits text deltas as they arrive", async () => {
  const client = {
    chat: {
      completions: {
        create: async () =>
          streamFrom([
            { choices: [{ delta: { role: "assistant" } }] },
            { choices: [{ delta: { content: "Hel" } }] },
            { choices: [{ delta: { content: "lo" } }] },
            {
              choices: [{ delta: {}, finish_reason: "stop" }],
              usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },
            },
          ]),
      },
    },
  };

  const events = await collectEvents(
    runToolAwareChatCompletionsStream({
      client: client as any,
      model: "grok-test",
      messages: [{ role: "user", content: "Say hello" }],
    })
  );

  assert.deepEqual(
    events.map((event) => event.type),
    ["delta", "delta", "done"]
  );
  assert.deepEqual(
    events.filter((event) => event.type === "delta").map((event) => event.text),
    ["Hel", "lo"]
  );
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hello");
});

test("plain Chat Completions stream does not send Sybil-managed tools", async () => {
  let requestBody: any = null;
  const client = {
    chat: {
      completions: {
        create: async (body: any) => {
          requestBody = body;
          return streamFrom([
            { choices: [{ delta: { content: "Hi" } }] },
            { choices: [{ delta: {}, finish_reason: "stop" }] },
          ]);
        },
      },
    },
  };

  const events = await collectEvents(
    runPlainChatCompletionsStream({
      client: client as any,
      model: "hermes-agent",
      messages: [{ role: "user", content: "Say hi" }],
    })
  );

  assert.equal(requestBody.model, "hermes-agent");
  assert.equal(requestBody.stream, true);
  assert.equal("tools" in requestBody, false);
  assert.deepEqual(
    events.map((event) => event.type),
    ["delta", "done"]
  );
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hi");
});

test("fetch_url sends browser-like navigation headers", async () => {
  const originalFetch = globalThis.fetch;
  const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
  globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
    fetchCalls.push({ input, init });
    return new Response("<!doctype html><title>CPI</title><main>Consumer price index</main>", {
      status: 200,
      headers: { "content-type": "text/html; charset=utf-8" },
    });
  }) as typeof fetch;

  try {
    let requestCount = 0;
    const client = {
      chat: {
        completions: {
          create: async () => {
            requestCount += 1;
            if (requestCount === 1) {
              return {
                choices: [
                  {
                    message: {
                      tool_calls: [
                        {
                          id: "call_1",
                          type: "function",
                          function: {
                            name: "fetch_url",
                            arguments: JSON.stringify({ url: "https://www.bls.gov/news.release/pdf/cpi.pdf" }),
                          },
                        },
                      ],
                    },
                  },
                ],
              };
            }

            return {
              choices: [{ message: { content: "Fetched" } }],
            };
          },
        },
      },
    };

    const result = await runToolAwareChatCompletions({
      client: client as any,
      model: "grok-test",
      messages: [{ role: "user", content: "Fetch CPI PDF" }],
    });

    assert.equal(result.text, "Fetched");
    assert.equal(fetchCalls.length, 1);
    assert.equal(String(fetchCalls[0]?.input), "https://www.bls.gov/news.release/pdf/cpi.pdf");
    assert.deepEqual(fetchCalls[0]?.init?.headers, {
      "User-Agent":
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,application/pdf;q=0.9,*/*;q=0.8",
      "Accept-Language": "en-US,en;q=0.9",
      "Upgrade-Insecure-Requests": "1",
      "Sec-Fetch-Dest": "document",
      "Sec-Fetch-Mode": "navigate",
      "Sec-Fetch-Site": "none",
      "Sec-Fetch-User": "?1",
    });
    assert.equal(result.toolEvents[0]?.status, "completed");
  } finally {
    globalThis.fetch = originalFetch;
  }
});

test("OpenAI-compatible Chat Completions stream emits initiated and terminal tool call updates", async () => {
  let requestCount = 0;
  const client = {
    chat: {
      completions: {
        create: async () => {
          requestCount += 1;
          if (requestCount === 1) {
            return streamFrom([
              {
                choices: [
                  {
                    delta: {
                      tool_calls: [
                        {
                          index: 0,
                          id: "call_1",
                          function: {
                            name: "unknown_tool",
                            arguments: "{\"query\":\"current weather\"}",
                          },
                        },
                      ],
                    },
                    finish_reason: "tool_calls",
                  },
                ],
              },
            ]);
          }

          return streamFrom([
            { choices: [{ delta: { content: "Done" } }] },
            { choices: [{ delta: {}, finish_reason: "stop" }] },
          ]);
        },
      },
    },
  };

  const events = await collectEvents(
    runToolAwareChatCompletionsStream({
      client: client as any,
      model: "grok-test",
      messages: [{ role: "user", content: "Use a tool" }],
    })
  );

  assert.deepEqual(
    events.map((event) => event.type),
    ["tool_call", "tool_call", "delta", "done"]
  );

  const toolEvents = events.flatMap((event) => (event.type === "tool_call" ? [event.event] : []));
  assert.equal(toolEvents[0]?.toolCallId, "call_1");
  assert.equal(toolEvents[0]?.status, "initiated");
  assert.equal(toolEvents[0]?.completedAt, undefined);
  assert.equal(toolEvents[0]?.durationMs, undefined);
  assert.equal(toolEvents[1]?.toolCallId, "call_1");
  assert.equal(toolEvents[1]?.status, "failed");
  assert.match(toolEvents[1]?.error ?? "", /Unknown tool: unknown_tool/);
  assert.equal(typeof toolEvents[1]?.completedAt, "string");
  assert.equal(typeof toolEvents[1]?.durationMs, "number");
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Done");
});