Add EasyList filter support

scripts/update-filter-lists.mjs

@@ -0,0 +1,98 @@
+#!/usr/bin/env node
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.join(__dirname, "..");
+const outputDir = path.join(repoRoot, "privacy-filters", "lists");
+
+const FILTER_LISTS = [
+  {
+    file: "easylist.txt",
+    url: "https://easylist.to/easylist/easylist.txt"
+  },
+  {
+    file: "ublock-filters.txt",
+    url: "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/filters.txt",
+    expandIncludes: true
+  },
+  {
+    file: "easylist-cookie.txt",
+    url: "https://easylist-downloads.adblockplus.org/fanboy-cookiemonster.txt"
+  },
+  {
+    file: "ublock-annoyances.txt",
+    url: "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/annoyances.txt",
+    expandIncludes: true
+  },
+  {
+    file: "ublock-cookies.txt",
+    url: "https://raw.githubusercontent.com/uBlockOrigin/uAssets/master/filters/annoyances-cookies.txt",
+    expandIncludes: true
+  }
+];
+
+async function main() {
+  await fs.mkdir(outputDir, { recursive: true });
+
+  for (const list of FILTER_LISTS) {
+    const content = list.expandIncludes
+      ? await fetchListWithIncludes(list.url)
+      : await fetchText(list.url);
+    const header = [
+      `! Bundled by scripts/update-filter-lists.mjs from ${list.url}`,
+      `! Bundled at ${new Date().toISOString()}`,
+      ""
+    ].join("\n");
+    const output = header + normalizeLineEndings(content).replace(/\s*$/, "\n");
+    const outputPath = path.join(outputDir, list.file);
+    await fs.writeFile(outputPath, output, "utf8");
+    console.log(`wrote ${path.relative(repoRoot, outputPath)} (${output.length} bytes)`);
+  }
+}
+
+async function fetchListWithIncludes(url, seen = new Set()) {
+  if (seen.has(url)) {
+    return "";
+  }
+  seen.add(url);
+
+  const content = await fetchText(url);
+  const lines = [];
+  for (const line of normalizeLineEndings(content).split("\n")) {
+    const includeMatch = line.match(/^!#include\s+(.+)$/);
+    if (!includeMatch) {
+      lines.push(line);
+      continue;
+    }
+
+    const includeUrl = new URL(includeMatch[1].trim(), url).href;
+    lines.push(`! >>> begin include ${includeMatch[1].trim()}`);
+    lines.push(await fetchListWithIncludes(includeUrl, seen));
+    lines.push(`! <<< end include ${includeMatch[1].trim()}`);
+  }
+  return lines.join("\n");
+}
+
+async function fetchText(url) {
+  const response = await fetch(url, {
+    headers: {
+      "user-agent": "local-page-archiver filter-list updater"
+    },
+    redirect: "follow"
+  });
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url}: HTTP ${response.status}`);
+  }
+  return response.text();
+}
+
+function normalizeLineEndings(value) {
+  return String(value).replace(/\r\n?/g, "\n");
+}
+
+main().catch((error) => {
+  console.error(error.message);
+  process.exitCode = 1;
+});