feat: extract text from fetched PDFs
This commit is contained in:
@@ -6,10 +6,12 @@ import { promisify } from "node:util";
|
||||
import { convert as htmlToText } from "html-to-text";
|
||||
import { z } from "zod";
|
||||
import { buildBrowserLikeNavigationHeaders } from "../browser-fetch-headers.js";
|
||||
import { MAX_FETCH_RESPONSE_BYTES, readBoundedResponseBytes } from "./bounded-response.js";
|
||||
import { env } from "../env.js";
|
||||
import { searchBrave } from "../search/brave.js";
|
||||
import { exaClient } from "../search/exa.js";
|
||||
import { searchSearxng } from "../search/searxng.js";
|
||||
import { extractPdfText, hasPdfMagic, MAX_PDF_PAGES_PER_FETCH } from "./pdf-text.js";
|
||||
import type { ChatMessage } from "./types.js";
|
||||
|
||||
export const MAX_TOOL_ROUNDS = env.CHAT_MAX_TOOL_ROUNDS;
|
||||
@@ -45,6 +47,7 @@ const FetchUrlArgsSchema = z
|
||||
.object({
|
||||
url: z.string().trim().url(),
|
||||
maxCharacters: z.coerce.number().int().min(500).max(MAX_FETCH_MAX_CHARACTERS).optional(),
|
||||
pages: z.array(z.coerce.number().int().min(1).max(1_000_000)).min(1).max(MAX_PDF_PAGES_PER_FETCH).optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
@@ -163,7 +166,7 @@ const BASE_CHAT_TOOLS: any[] = [
|
||||
function: {
|
||||
name: "fetch_url",
|
||||
description:
|
||||
"Fetch a webpage by URL and return readable plaintext content extracted from the page for deeper inspection.",
|
||||
"Fetch a webpage by URL and return readable plaintext content. Text PDFs are extracted page by page; use pages:[3,4] to retrieve selected pages after a truncated result. Scanned PDFs have no extractable text.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
@@ -174,6 +177,12 @@ const BASE_CHAT_TOOLS: any[] = [
|
||||
maximum: MAX_FETCH_MAX_CHARACTERS,
|
||||
description: "Maximum response text characters returned (default 12000).",
|
||||
},
|
||||
pages: {
|
||||
type: "array",
|
||||
items: { type: "integer", minimum: 1, maximum: 1000000 },
|
||||
maxItems: MAX_PDF_PAGES_PER_FETCH,
|
||||
description: "For PDFs, extract these page numbers only (up to 30 per fetch).",
|
||||
},
|
||||
},
|
||||
required: ["url"],
|
||||
additionalProperties: false,
|
||||
@@ -561,48 +570,69 @@ async function runFetchUrlTool(input: unknown): Promise<ToolRunOutcome> {
|
||||
signal: controller.signal,
|
||||
headers: buildBrowserLikeNavigationHeaders(),
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
throw new Error(`Fetch failed with status ${response.status}.`);
|
||||
}
|
||||
|
||||
const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
|
||||
const bytes = await readBoundedResponseBytes(response, MAX_FETCH_RESPONSE_BYTES);
|
||||
const url = response.url || parsed.toString();
|
||||
if (contentType.includes("application/pdf") || hasPdfMagic(bytes)) {
|
||||
if (!hasPdfMagic(bytes)) {
|
||||
return { ok: false, url, status: response.status, contentType: contentType || null, pdfStatus: "invalid", text: "The server declared a PDF, but the downloaded file does not have a valid PDF header." };
|
||||
}
|
||||
const result = await extractPdfText(bytes, args.pages, maxCharacters);
|
||||
const text = result.truncated
|
||||
? `${result.text}\n\n[truncated; re-fetch selected pages with pages:[N,...]]`
|
||||
: result.text;
|
||||
return {
|
||||
ok: result.status === "ok",
|
||||
url,
|
||||
status: response.status,
|
||||
contentType: contentType || "application/pdf",
|
||||
pdfStatus: result.status,
|
||||
pageCount: result.pageCount,
|
||||
pages: result.pages,
|
||||
pagesTruncated: result.pageCount !== null && result.pageCount > result.pages.length,
|
||||
truncated: result.truncated,
|
||||
text,
|
||||
};
|
||||
}
|
||||
|
||||
const body = new TextDecoder().decode(bytes);
|
||||
const isHtml = contentType.includes("text/html") || /<!doctype html|<html[\s>]/i.test(body);
|
||||
const extracted = isHtml
|
||||
? htmlToText(body, {
|
||||
wordwrap: false,
|
||||
preserveNewlines: true,
|
||||
selectors: [
|
||||
{ selector: "img", format: "skip" },
|
||||
{ selector: "script", format: "skip" },
|
||||
{ selector: "style", format: "skip" },
|
||||
{ selector: "noscript", format: "skip" },
|
||||
{ selector: "a", options: { ignoreHref: true } },
|
||||
],
|
||||
})
|
||||
: body;
|
||||
const normalized = compactWhitespace(extracted);
|
||||
const truncated = normalized.length > maxCharacters;
|
||||
const text = truncated
|
||||
? `${normalized.slice(0, maxCharacters)}\n\n[truncated ${normalized.length - maxCharacters} characters]`
|
||||
: normalized;
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
url,
|
||||
status: response.status,
|
||||
contentType: contentType || null,
|
||||
title: isHtml ? extractHtmlTitle(body) : null,
|
||||
truncated,
|
||||
text,
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Fetch failed with status ${response.status}.`);
|
||||
}
|
||||
|
||||
const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
|
||||
const body = await response.text();
|
||||
const isHtml = contentType.includes("text/html") || /<!doctype html|<html[\s>]/i.test(body);
|
||||
|
||||
let extracted = body;
|
||||
if (isHtml) {
|
||||
extracted = htmlToText(body, {
|
||||
wordwrap: false,
|
||||
preserveNewlines: true,
|
||||
selectors: [
|
||||
{ selector: "img", format: "skip" },
|
||||
{ selector: "script", format: "skip" },
|
||||
{ selector: "style", format: "skip" },
|
||||
{ selector: "noscript", format: "skip" },
|
||||
{ selector: "a", options: { ignoreHref: true } },
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
const normalized = compactWhitespace(extracted);
|
||||
const truncated = normalized.length > maxCharacters;
|
||||
const text = truncated
|
||||
? `${normalized.slice(0, maxCharacters)}\n\n[truncated ${normalized.length - maxCharacters} characters]`
|
||||
: normalized;
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
url: response.url || parsed.toString(),
|
||||
status: response.status,
|
||||
contentType: contentType || null,
|
||||
title: isHtml ? extractHtmlTitle(body) : null,
|
||||
truncated,
|
||||
text,
|
||||
};
|
||||
}
|
||||
|
||||
function shellQuote(value: string) {
|
||||
|
||||
Reference in New Issue
Block a user