feat: extract text from fetched PDFs

This commit is contained in:
2026-11-01 14:20:30 -08:00
parent a390c7287a
commit cb9d5a787b
13 changed files with 768 additions and 43 deletions
+70 -40
View File
@@ -6,10 +6,12 @@ import { promisify } from "node:util";
import { convert as htmlToText } from "html-to-text";
import { z } from "zod";
import { buildBrowserLikeNavigationHeaders } from "../browser-fetch-headers.js";
import { MAX_FETCH_RESPONSE_BYTES, readBoundedResponseBytes } from "./bounded-response.js";
import { env } from "../env.js";
import { searchBrave } from "../search/brave.js";
import { exaClient } from "../search/exa.js";
import { searchSearxng } from "../search/searxng.js";
import { extractPdfText, hasPdfMagic, MAX_PDF_PAGES_PER_FETCH } from "./pdf-text.js";
import type { ChatMessage } from "./types.js";
export const MAX_TOOL_ROUNDS = env.CHAT_MAX_TOOL_ROUNDS;
@@ -45,6 +47,7 @@ const FetchUrlArgsSchema = z
.object({
url: z.string().trim().url(),
maxCharacters: z.coerce.number().int().min(500).max(MAX_FETCH_MAX_CHARACTERS).optional(),
pages: z.array(z.coerce.number().int().min(1).max(1_000_000)).min(1).max(MAX_PDF_PAGES_PER_FETCH).optional(),
})
.strict();
@@ -163,7 +166,7 @@ const BASE_CHAT_TOOLS: any[] = [
function: {
name: "fetch_url",
description:
"Fetch a webpage by URL and return readable plaintext content extracted from the page for deeper inspection.",
"Fetch a webpage by URL and return readable plaintext content. Text PDFs are extracted page by page; use pages:[3,4] to retrieve selected pages after a truncated result. Scanned PDFs have no extractable text.",
parameters: {
type: "object",
properties: {
@@ -174,6 +177,12 @@ const BASE_CHAT_TOOLS: any[] = [
maximum: MAX_FETCH_MAX_CHARACTERS,
description: "Maximum response text characters returned (default 12000).",
},
pages: {
type: "array",
items: { type: "integer", minimum: 1, maximum: 1000000 },
maxItems: MAX_PDF_PAGES_PER_FETCH,
description: "For PDFs, extract these page numbers only (up to 30 per fetch).",
},
},
required: ["url"],
additionalProperties: false,
@@ -561,48 +570,69 @@ async function runFetchUrlTool(input: unknown): Promise<ToolRunOutcome> {
signal: controller.signal,
headers: buildBrowserLikeNavigationHeaders(),
});
if (!response.ok) {
await response.body?.cancel();
throw new Error(`Fetch failed with status ${response.status}.`);
}
const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
const bytes = await readBoundedResponseBytes(response, MAX_FETCH_RESPONSE_BYTES);
const url = response.url || parsed.toString();
if (contentType.includes("application/pdf") || hasPdfMagic(bytes)) {
if (!hasPdfMagic(bytes)) {
return { ok: false, url, status: response.status, contentType: contentType || null, pdfStatus: "invalid", text: "The server declared a PDF, but the downloaded file does not have a valid PDF header." };
}
const result = await extractPdfText(bytes, args.pages, maxCharacters);
const text = result.truncated
? `${result.text}\n\n[truncated; re-fetch selected pages with pages:[N,...]]`
: result.text;
return {
ok: result.status === "ok",
url,
status: response.status,
contentType: contentType || "application/pdf",
pdfStatus: result.status,
pageCount: result.pageCount,
pages: result.pages,
pagesTruncated: result.pageCount !== null && result.pageCount > result.pages.length,
truncated: result.truncated,
text,
};
}
const body = new TextDecoder().decode(bytes);
const isHtml = contentType.includes("text/html") || /<!doctype html|<html[\s>]/i.test(body);
const extracted = isHtml
? htmlToText(body, {
wordwrap: false,
preserveNewlines: true,
selectors: [
{ selector: "img", format: "skip" },
{ selector: "script", format: "skip" },
{ selector: "style", format: "skip" },
{ selector: "noscript", format: "skip" },
{ selector: "a", options: { ignoreHref: true } },
],
})
: body;
const normalized = compactWhitespace(extracted);
const truncated = normalized.length > maxCharacters;
const text = truncated
? `${normalized.slice(0, maxCharacters)}\n\n[truncated ${normalized.length - maxCharacters} characters]`
: normalized;
return {
ok: true,
url,
status: response.status,
contentType: contentType || null,
title: isHtml ? extractHtmlTitle(body) : null,
truncated,
text,
};
} finally {
clearTimeout(timeout);
}
if (!response.ok) {
throw new Error(`Fetch failed with status ${response.status}.`);
}
const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
const body = await response.text();
const isHtml = contentType.includes("text/html") || /<!doctype html|<html[\s>]/i.test(body);
let extracted = body;
if (isHtml) {
extracted = htmlToText(body, {
wordwrap: false,
preserveNewlines: true,
selectors: [
{ selector: "img", format: "skip" },
{ selector: "script", format: "skip" },
{ selector: "style", format: "skip" },
{ selector: "noscript", format: "skip" },
{ selector: "a", options: { ignoreHref: true } },
],
});
}
const normalized = compactWhitespace(extracted);
const truncated = normalized.length > maxCharacters;
const text = truncated
? `${normalized.slice(0, maxCharacters)}\n\n[truncated ${normalized.length - maxCharacters} characters]`
: normalized;
return {
ok: true,
url: response.url || parsed.toString(),
status: response.status,
contentType: contentType || null,
title: isHtml ? extractHtmlTitle(body) : null,
truncated,
text,
};
}
function shellQuote(value: string) {