adds attachment support

server/src/llm/message-content.ts

@@ -0,0 +1,211 @@
+import type { ChatAttachment, ChatImageAttachment, ChatMessage, ChatTextAttachment } from "./types.js";
+
+function escapeAttribute(value: string) {
+  return value.replace(/"/g, """);
+}
+
+function getImageAttachments(message: ChatMessage) {
+  return (message.attachments ?? []).filter((attachment): attachment is ChatImageAttachment => attachment.kind === "image");
+}
+
+function getTextAttachments(message: ChatMessage) {
+  return (message.attachments ?? []).filter((attachment): attachment is ChatTextAttachment => attachment.kind === "text");
+}
+
+function buildImageSummaryText(attachments: ChatImageAttachment[]) {
+  if (!attachments.length) return null;
+  const label = attachments.length === 1 ? "Attached image" : "Attached images";
+  return `${label}: ${attachments.map((attachment) => attachment.filename).join(", ")}.`;
+}
+
+function buildTextAttachmentPrompt(attachment: ChatTextAttachment) {
+  const truncationNote = attachment.truncated ? ' truncated="true"' : "";
+  return [
+    `Attached text file: ${attachment.filename}${attachment.truncated ? " (content truncated)" : ""}`,
+    `<attached_file filename="${escapeAttribute(attachment.filename)}" mime_type="${escapeAttribute(attachment.mimeType)}"${truncationNote}>`,
+    attachment.text,
+    "</attached_file>",
+  ].join("\n");
+}
+
+function toOpenAIContent(message: ChatMessage) {
+  const imageAttachments = getImageAttachments(message);
+  const textAttachments = getTextAttachments(message);
+  if (!imageAttachments.length && !textAttachments.length) {
+    return message.content;
+  }
+
+  const parts: Array<Record<string, unknown>> = [];
+
+  for (const attachment of imageAttachments) {
+    parts.push({
+      type: "image_url",
+      image_url: {
+        url: attachment.dataUrl,
+        detail: "auto",
+      },
+    });
+  }
+
+  const imageSummary = buildImageSummaryText(imageAttachments);
+  if (imageSummary) {
+    parts.push({ type: "text", text: imageSummary });
+  }
+
+  for (const attachment of textAttachments) {
+    parts.push({ type: "text", text: buildTextAttachmentPrompt(attachment) });
+  }
+
+  if (message.content.trim()) {
+    parts.push({ type: "text", text: message.content });
+  }
+
+  if (parts.length === 1 && parts[0]?.type === "text" && typeof parts[0].text === "string") {
+    return parts[0].text;
+  }
+
+  return parts;
+}
+
+function parseImageDataUrl(attachment: ChatImageAttachment) {
+  const match = attachment.dataUrl.match(/^data:(image\/(?:png|jpeg));base64,([a-z0-9+/=\s]+)$/i);
+  if (!match) {
+    throw new Error(`Invalid image attachment data URL for '${attachment.filename}'.`);
+  }
+
+  const mediaType = match[1].toLowerCase();
+  if (mediaType !== attachment.mimeType) {
+    throw new Error(`Image attachment MIME type mismatch for '${attachment.filename}'.`);
+  }
+
+  return {
+    mediaType,
+    data: match[2].replace(/\s+/g, ""),
+  };
+}
+
+function toAnthropicContent(message: ChatMessage) {
+  const imageAttachments = getImageAttachments(message);
+  const textAttachments = getTextAttachments(message);
+  if (!imageAttachments.length && !textAttachments.length) {
+    return message.content;
+  }
+
+  const blocks: Array<Record<string, unknown>> = [];
+
+  for (const attachment of imageAttachments) {
+    const source = parseImageDataUrl(attachment);
+    blocks.push({
+      type: "image",
+      source: {
+        type: "base64",
+        media_type: source.mediaType,
+        data: source.data,
+      },
+    });
+  }
+
+  const imageSummary = buildImageSummaryText(imageAttachments);
+  if (imageSummary) {
+    blocks.push({ type: "text", text: imageSummary });
+  }
+
+  for (const attachment of textAttachments) {
+    blocks.push({ type: "text", text: buildTextAttachmentPrompt(attachment) });
+  }
+
+  if (message.content.trim()) {
+    blocks.push({ type: "text", text: message.content });
+  }
+
+  if (blocks.length === 1 && blocks[0]?.type === "text" && typeof blocks[0].text === "string") {
+    return blocks[0].text;
+  }
+
+  return blocks;
+}
+
+export function buildOpenAIConversationMessage(message: ChatMessage) {
+  if (message.role === "tool") {
+    const name = message.name?.trim() || "tool";
+    return {
+      role: "user",
+      content: `Tool output (${name}):\n${message.content}`,
+    };
+  }
+
+  const out: Record<string, unknown> = {
+    role: message.role,
+    content: toOpenAIContent(message),
+  };
+
+  if (message.name && (message.role === "assistant" || message.role === "user")) {
+    out.name = message.name;
+  }
+
+  return out;
+}
+
+export function getAnthropicSystemPrompt(messages: ChatMessage[]) {
+  return messages.find((message) => message.role === "system")?.content;
+}
+
+export function buildAnthropicConversationMessage(message: ChatMessage) {
+  if (message.role === "system") {
+    throw new Error("System messages must be handled separately for Anthropic.");
+  }
+
+  if (message.role === "tool") {
+    const name = message.name?.trim() || "tool";
+    return {
+      role: "user",
+      content: `Tool output (${name}):\n${message.content}`,
+    };
+  }
+
+  return {
+    role: message.role === "assistant" ? "assistant" : "user",
+    content: toAnthropicContent(message),
+  };
+}
+
+export function buildComparableAttachments(input: unknown): ChatAttachment[] {
+  if (!Array.isArray(input)) return [];
+
+  const attachments: ChatAttachment[] = [];
+  for (const entry of input) {
+    if (!entry || typeof entry !== "object" || Array.isArray(entry)) continue;
+    const record = entry as Record<string, unknown>;
+    const kind = record.kind;
+    const id = typeof record.id === "string" ? record.id : "";
+    const filename = typeof record.filename === "string" ? record.filename : "";
+    const mimeType = typeof record.mimeType === "string" ? record.mimeType : "";
+    const sizeBytes = typeof record.sizeBytes === "number" ? record.sizeBytes : 0;
+
+    if (kind === "image" && typeof record.dataUrl === "string") {
+      attachments.push({
+        kind,
+        id,
+        filename,
+        mimeType: mimeType === "image/png" ? "image/png" : "image/jpeg",
+        sizeBytes,
+        dataUrl: record.dataUrl,
+      });
+      continue;
+    }
+
+    if (kind === "text" && typeof record.text === "string") {
+      attachments.push({
+        kind,
+        id,
+        filename,
+        mimeType,
+        sizeBytes,
+        text: record.text,
+        truncated: record.truncated === true,
+      });
+    }
+  }
+
+  return attachments;
+}