fix most web_fetches from getting blocked using a real user agent

2026-06-11 23:36:19 -07:00
parent 22aa652257
commit d7214c88ad
8 changed files with 403 additions and 54 deletions
--- a/server/tests/chat-tools-streaming.test.ts
+++ b/server/tests/chat-tools-streaming.test.ts
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
 import test from "node:test";
 import {
  runPlainChatCompletionsStream,
+  runToolAwareChatCompletions,
  runToolAwareChatCompletionsStream,
  runToolAwareOpenAIChatStream,
  type ToolAwareStreamingEvent,
@@ -141,6 +142,79 @@ test("plain Chat Completions stream does not send Sybil-managed tools", async ()
  assert.equal(events.at(-1)?.type === "done" ? events.at(-1)?.result.text : null, "Hi");
 });

+test("fetch_url sends browser-like navigation headers", async () => {
+  const originalFetch = globalThis.fetch;
+  const fetchCalls: Array<{ input: RequestInfo | URL; init?: RequestInit }> = [];
+  globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+    fetchCalls.push({ input, init });
+    return new Response("<!doctype html><title>CPI</title><main>Consumer price index</main>", {
+      status: 200,
+      headers: { "content-type": "text/html; charset=utf-8" },
+    });
+  }) as typeof fetch;
+
+  try {
+    let requestCount = 0;
+    const client = {
+      chat: {
+        completions: {
+          create: async () => {
+            requestCount += 1;
+            if (requestCount === 1) {
+              return {
+                choices: [
+                  {
+                    message: {
+                      tool_calls: [
+                        {
+                          id: "call_1",
+                          type: "function",
+                          function: {
+                            name: "fetch_url",
+                            arguments: JSON.stringify({ url: "https://www.bls.gov/news.release/pdf/cpi.pdf" }),
+                          },
+                        },
+                      ],
+                    },
+                  },
+                ],
+              };
+            }
+
+            return {
+              choices: [{ message: { content: "Fetched" } }],
+            };
+          },
+        },
+      },
+    };
+
+    const result = await runToolAwareChatCompletions({
+      client: client as any,
+      model: "grok-test",
+      messages: [{ role: "user", content: "Fetch CPI PDF" }],
+    });
+
+    assert.equal(result.text, "Fetched");
+    assert.equal(fetchCalls.length, 1);
+    assert.equal(String(fetchCalls[0]?.input), "https://www.bls.gov/news.release/pdf/cpi.pdf");
+    assert.deepEqual(fetchCalls[0]?.init?.headers, {
+      "User-Agent":
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
+      Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,application/pdf;q=0.9,*/*;q=0.8",
+      "Accept-Language": "en-US,en;q=0.9",
+      "Upgrade-Insecure-Requests": "1",
+      "Sec-Fetch-Dest": "document",
+      "Sec-Fetch-Mode": "navigate",
+      "Sec-Fetch-Site": "none",
+      "Sec-Fetch-User": "?1",
+    });
+    assert.equal(result.toolEvents[0]?.status, "completed");
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
 test("OpenAI-compatible Chat Completions stream emits initiated and terminal tool call updates", async () => {
  let requestCount = 0;
  const client = {