diff --git a/dist/default.conf b/dist/default.conf index db8a43a..d50b55c 100644 --- a/dist/default.conf +++ b/dist/default.conf @@ -1,6 +1,7 @@ server { listen 80; server_name _; + client_max_body_size 32m; root /usr/share/nginx/html; index index.html; diff --git a/docs/api/rest.md b/docs/api/rest.md index 42ca7ab..75ceda3 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -10,6 +10,12 @@ Content type: - Requests with bodies use `application/json`. - Responses are JSON unless noted otherwise. +Chat upload limits: +- Chat completion and direct message payloads support inline attachments up to a 32 MB request body. +- Up to 8 attachments per message. +- Image attachments: PNG or JPEG only, max 6 MB each. +- Text attachments: up to 8 MB source size each; server accepts at most 200,000 characters of inlined text content per attachment. + ## Health + Auth ### `GET /health` @@ -74,11 +80,34 @@ Behavior notes: "role": "system|user|assistant|tool", "content": "string", "name": "optional", - "metadata": {} + "metadata": {}, + "attachments": [ + { + "kind": "image", + "id": "attachment-id", + "filename": "photo.jpg", + "mimeType": "image/jpeg", + "sizeBytes": 12345, + "dataUrl": "data:image/jpeg;base64,..." + }, + { + "kind": "text", + "id": "attachment-id", + "filename": "notes.md", + "mimeType": "text/markdown", + "sizeBytes": 4567, + "text": "# Notes\\n...", + "truncated": false + } + ] } ``` - Response: `{ "message": Message }` +Notes: +- `attachments` is optional and is merged into stored `message.metadata.attachments`. +- Tool messages should not include attachments. + ## Chat Completions (non-streaming) ### `POST /v1/chat-completions` @@ -89,7 +118,30 @@ Behavior notes: "provider": "openai|anthropic|xai", "model": "string", "messages": [ - { "role": "system|user|assistant|tool", "content": "string", "name": "optional" } + { + "role": "system|user|assistant|tool", + "content": "string", + "name": "optional", + "attachments": [ + { + "kind": "image", + "id": "attachment-id", + "filename": "photo.jpg", + "mimeType": "image/jpeg", + "sizeBytes": 12345, + "dataUrl": "data:image/jpeg;base64,..." + }, + { + "kind": "text", + "id": "attachment-id", + "filename": "notes.md", + "mimeType": "text/markdown", + "sizeBytes": 4567, + "text": "# Notes\\n...", + "truncated": false + } + ] + } ], "temperature": 0.2, "maxTokens": 256 @@ -112,7 +164,12 @@ Behavior notes: - For `chatId` calls, server stores only *new* non-assistant messages from provided history to avoid duplicates. - Server persists final assistant output and call metadata (`LlmCall`) in DB. - Server updates chat-level model metadata on each call: `lastUsedProvider`/`lastUsedModel`; first successful/failed call also initializes `initiatedProvider`/`initiatedModel` if unset. +- Attachments are optional and currently apply to `user` messages. Persisted chat history stores them under `message.metadata.attachments`. +- Images are forwarded inline to providers as multimodal image parts. Use PNG or JPEG for cross-provider compatibility. +- Text files are forwarded as explicit text blocks rather than provider-managed file references. Large text attachments should already be truncated client-side before submission. - For `openai` and `xai`, backend enables tool use during chat completion with an internal system instruction. +- For `openai` and `xai`, image attachments are sent as chat-completions content parts alongside text. +- For `anthropic`, image attachments are sent as Messages API `image` blocks using base64 source data; text attachments are added as `text` blocks. - Available tool calls for chat: `web_search` and `fetch_url`. - `web_search` returns ranked results with per-result summaries/snippets. Its backend engine is selected by `CHAT_WEB_SEARCH_ENGINE` (`exa` default, or `searxng` with `SEARXNG_BASE_URL` set). SearXNG mode requires the instance to allow `format=json`. - `fetch_url` fetches a URL and returns plaintext page content (HTML converted to text server-side). @@ -189,10 +246,32 @@ Search run notes: "role": "system|user|assistant|tool", "content": "...", "name": null, - "metadata": null + "metadata": { + "attachments": [ + { + "kind": "image", + "id": "attachment-id", + "filename": "photo.jpg", + "mimeType": "image/jpeg", + "sizeBytes": 12345, + "dataUrl": "data:image/jpeg;base64,..." + }, + { + "kind": "text", + "id": "attachment-id", + "filename": "notes.md", + "mimeType": "text/markdown", + "sizeBytes": 4567, + "text": "# Notes\\n...", + "truncated": false + } + ] + } } ``` +`metadata` remains nullable. Tool-call log messages still use `metadata.kind = "tool_call"`; regular user messages with attachments use `metadata.attachments`. + `ChatDetail` ```json { diff --git a/docs/api/streaming-chat.md b/docs/api/streaming-chat.md index e5e8826..1b10490 100644 --- a/docs/api/streaming-chat.md +++ b/docs/api/streaming-chat.md @@ -9,6 +9,7 @@ Transport: - HTTP response uses `Content-Type: text/event-stream; charset=utf-8` - Events are emitted in SSE format (`event: ...`, `data: ...`) - Request body is JSON +- Request body supports the same inline attachment schema and limits documented in `docs/api/rest.md`. Authentication: - Same as REST endpoints (`Authorization: Bearer ` when token mode is enabled) @@ -21,7 +22,30 @@ Authentication: "provider": "openai|anthropic|xai", "model": "string", "messages": [ - { "role": "system|user|assistant|tool", "content": "string", "name": "optional" } + { + "role": "system|user|assistant|tool", + "content": "string", + "name": "optional", + "attachments": [ + { + "kind": "image", + "id": "attachment-id", + "filename": "photo.jpg", + "mimeType": "image/jpeg", + "sizeBytes": 12345, + "dataUrl": "data:image/jpeg;base64,..." + }, + { + "kind": "text", + "id": "attachment-id", + "filename": "notes.md", + "mimeType": "text/markdown", + "sizeBytes": 4567, + "text": "# Notes\\n...", + "truncated": false + } + ] + } ], "temperature": 0.2, "maxTokens": 256 @@ -32,6 +56,7 @@ Notes: - If `chatId` is omitted, backend creates a new chat. - If `chatId` is provided, backend validates it exists. - Backend stores only new non-assistant input history rows to avoid duplicates. +- Attachments are optional and are persisted under `message.metadata.attachments` on stored user messages. ## Event Stream Contract @@ -103,8 +128,9 @@ Event order: ## Provider Streaming Behavior - `openai`: backend may execute internal tool calls (`web_search`, `fetch_url`) before producing final text. +- `openai`: image attachments are sent as chat-completions content parts; text attachments are inlined as text parts. - `xai`: same tool-enabled behavior as OpenAI. -- `anthropic`: streamed via event stream; emits `delta` from `content_block_delta` with `text_delta`. +- `anthropic`: streamed via event stream; emits `delta` from `content_block_delta` with `text_delta`. Image attachments are sent as base64 `image` blocks and text attachments are appended as `text` blocks. - `web_search` uses `CHAT_WEB_SEARCH_ENGINE` (`exa` default, or `searxng` with `SEARXNG_BASE_URL` set). SearXNG mode requires the instance to allow `format=json`. This only affects chat-mode tool calls, not search-mode endpoints. Tool-enabled streaming notes (`openai`/`xai`): diff --git a/server/src/index.ts b/server/src/index.ts index 45662f4..e7c52bc 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -9,6 +9,7 @@ import { warmModelCatalog } from "./llm/model-catalog.js"; import { registerRoutes } from "./routes.js"; const app = Fastify({ + bodyLimit: 32 * 1024 * 1024, disableRequestLogging: true, logger: { transport: { diff --git a/server/src/llm/chat-tools.ts b/server/src/llm/chat-tools.ts index d96dd3c..8a365b7 100644 --- a/server/src/llm/chat-tools.ts +++ b/server/src/llm/chat-tools.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import { env } from "../env.js"; import { exaClient } from "../search/exa.js"; import { searchSearxng } from "../search/searxng.js"; +import { buildOpenAIConversationMessage } from "./message-content.js"; import type { ChatMessage } from "./types.js"; const MAX_TOOL_ROUNDS = 4; @@ -250,23 +251,7 @@ function extractHtmlTitle(html: string) { } function normalizeIncomingMessages(messages: ChatMessage[]) { - const normalized = messages.map((m) => { - if (m.role === "tool") { - const name = m.name?.trim() || "tool"; - return { - role: "user", - content: `Tool output (${name}):\n${m.content}`, - }; - } - if (m.role === "assistant" || m.role === "system" || m.role === "user") { - const out: any = { role: m.role, content: m.content }; - if (m.name && (m.role === "assistant" || m.role === "user")) { - out.name = m.name; - } - return out; - } - return { role: "user", content: m.content }; - }); + const normalized = messages.map((message) => buildOpenAIConversationMessage(message)); return [{ role: "system", content: CHAT_TOOL_SYSTEM_PROMPT }, ...normalized]; } diff --git a/server/src/llm/message-content.ts b/server/src/llm/message-content.ts new file mode 100644 index 0000000..45cd32b --- /dev/null +++ b/server/src/llm/message-content.ts @@ -0,0 +1,211 @@ +import type { ChatAttachment, ChatImageAttachment, ChatMessage, ChatTextAttachment } from "./types.js"; + +function escapeAttribute(value: string) { + return value.replace(/"/g, """); +} + +function getImageAttachments(message: ChatMessage) { + return (message.attachments ?? []).filter((attachment): attachment is ChatImageAttachment => attachment.kind === "image"); +} + +function getTextAttachments(message: ChatMessage) { + return (message.attachments ?? []).filter((attachment): attachment is ChatTextAttachment => attachment.kind === "text"); +} + +function buildImageSummaryText(attachments: ChatImageAttachment[]) { + if (!attachments.length) return null; + const label = attachments.length === 1 ? "Attached image" : "Attached images"; + return `${label}: ${attachments.map((attachment) => attachment.filename).join(", ")}.`; +} + +function buildTextAttachmentPrompt(attachment: ChatTextAttachment) { + const truncationNote = attachment.truncated ? ' truncated="true"' : ""; + return [ + `Attached text file: ${attachment.filename}${attachment.truncated ? " (content truncated)" : ""}`, + ``, + attachment.text, + "", + ].join("\n"); +} + +function toOpenAIContent(message: ChatMessage) { + const imageAttachments = getImageAttachments(message); + const textAttachments = getTextAttachments(message); + if (!imageAttachments.length && !textAttachments.length) { + return message.content; + } + + const parts: Array> = []; + + for (const attachment of imageAttachments) { + parts.push({ + type: "image_url", + image_url: { + url: attachment.dataUrl, + detail: "auto", + }, + }); + } + + const imageSummary = buildImageSummaryText(imageAttachments); + if (imageSummary) { + parts.push({ type: "text", text: imageSummary }); + } + + for (const attachment of textAttachments) { + parts.push({ type: "text", text: buildTextAttachmentPrompt(attachment) }); + } + + if (message.content.trim()) { + parts.push({ type: "text", text: message.content }); + } + + if (parts.length === 1 && parts[0]?.type === "text" && typeof parts[0].text === "string") { + return parts[0].text; + } + + return parts; +} + +function parseImageDataUrl(attachment: ChatImageAttachment) { + const match = attachment.dataUrl.match(/^data:(image\/(?:png|jpeg));base64,([a-z0-9+/=\s]+)$/i); + if (!match) { + throw new Error(`Invalid image attachment data URL for '${attachment.filename}'.`); + } + + const mediaType = match[1].toLowerCase(); + if (mediaType !== attachment.mimeType) { + throw new Error(`Image attachment MIME type mismatch for '${attachment.filename}'.`); + } + + return { + mediaType, + data: match[2].replace(/\s+/g, ""), + }; +} + +function toAnthropicContent(message: ChatMessage) { + const imageAttachments = getImageAttachments(message); + const textAttachments = getTextAttachments(message); + if (!imageAttachments.length && !textAttachments.length) { + return message.content; + } + + const blocks: Array> = []; + + for (const attachment of imageAttachments) { + const source = parseImageDataUrl(attachment); + blocks.push({ + type: "image", + source: { + type: "base64", + media_type: source.mediaType, + data: source.data, + }, + }); + } + + const imageSummary = buildImageSummaryText(imageAttachments); + if (imageSummary) { + blocks.push({ type: "text", text: imageSummary }); + } + + for (const attachment of textAttachments) { + blocks.push({ type: "text", text: buildTextAttachmentPrompt(attachment) }); + } + + if (message.content.trim()) { + blocks.push({ type: "text", text: message.content }); + } + + if (blocks.length === 1 && blocks[0]?.type === "text" && typeof blocks[0].text === "string") { + return blocks[0].text; + } + + return blocks; +} + +export function buildOpenAIConversationMessage(message: ChatMessage) { + if (message.role === "tool") { + const name = message.name?.trim() || "tool"; + return { + role: "user", + content: `Tool output (${name}):\n${message.content}`, + }; + } + + const out: Record = { + role: message.role, + content: toOpenAIContent(message), + }; + + if (message.name && (message.role === "assistant" || message.role === "user")) { + out.name = message.name; + } + + return out; +} + +export function getAnthropicSystemPrompt(messages: ChatMessage[]) { + return messages.find((message) => message.role === "system")?.content; +} + +export function buildAnthropicConversationMessage(message: ChatMessage) { + if (message.role === "system") { + throw new Error("System messages must be handled separately for Anthropic."); + } + + if (message.role === "tool") { + const name = message.name?.trim() || "tool"; + return { + role: "user", + content: `Tool output (${name}):\n${message.content}`, + }; + } + + return { + role: message.role === "assistant" ? "assistant" : "user", + content: toAnthropicContent(message), + }; +} + +export function buildComparableAttachments(input: unknown): ChatAttachment[] { + if (!Array.isArray(input)) return []; + + const attachments: ChatAttachment[] = []; + for (const entry of input) { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) continue; + const record = entry as Record; + const kind = record.kind; + const id = typeof record.id === "string" ? record.id : ""; + const filename = typeof record.filename === "string" ? record.filename : ""; + const mimeType = typeof record.mimeType === "string" ? record.mimeType : ""; + const sizeBytes = typeof record.sizeBytes === "number" ? record.sizeBytes : 0; + + if (kind === "image" && typeof record.dataUrl === "string") { + attachments.push({ + kind, + id, + filename, + mimeType: mimeType === "image/png" ? "image/png" : "image/jpeg", + sizeBytes, + dataUrl: record.dataUrl, + }); + continue; + } + + if (kind === "text" && typeof record.text === "string") { + attachments.push({ + kind, + id, + filename, + mimeType, + sizeBytes, + text: record.text, + truncated: record.truncated === true, + }); + } + } + + return attachments; +} diff --git a/server/src/llm/multiplexer.ts b/server/src/llm/multiplexer.ts index 324df0e..9f150d5 100644 --- a/server/src/llm/multiplexer.ts +++ b/server/src/llm/multiplexer.ts @@ -2,6 +2,7 @@ import { performance } from "node:perf_hooks"; import { prisma } from "../db.js"; import { anthropicClient, openaiClient, xaiClient } from "./providers.js"; import { buildToolLogMessageData, runToolAwareOpenAIChat } from "./chat-tools.js"; +import { buildAnthropicConversationMessage, getAnthropicSystemPrompt } from "./message-content.js"; import type { MultiplexRequest, MultiplexResponse, Provider } from "./types.js"; function asProviderEnum(p: Provider) { @@ -68,11 +69,8 @@ export async function runMultiplex(req: MultiplexRequest): Promise m.role === "system")?.content; - const msgs = req.messages - .filter((m) => m.role !== "system") - .map((m) => ({ role: m.role === "assistant" ? "assistant" : "user", content: m.content })); + const system = getAnthropicSystemPrompt(req.messages); + const msgs = req.messages.filter((message) => message.role !== "system").map((message) => buildAnthropicConversationMessage(message)); const r = await client.messages.create({ model: req.model, diff --git a/server/src/llm/streaming.ts b/server/src/llm/streaming.ts index fb54c68..434a2ed 100644 --- a/server/src/llm/streaming.ts +++ b/server/src/llm/streaming.ts @@ -2,6 +2,7 @@ import { performance } from "node:perf_hooks"; import { prisma } from "../db.js"; import { anthropicClient, openaiClient, xaiClient } from "./providers.js"; import { buildToolLogMessageData, runToolAwareOpenAIChatStream, type ToolExecutionEvent } from "./chat-tools.js"; +import { buildAnthropicConversationMessage, getAnthropicSystemPrompt } from "./message-content.js"; import type { MultiplexRequest, Provider } from "./types.js"; export type StreamEvent = @@ -88,10 +89,8 @@ export async function* runMultiplexStream(req: MultiplexRequest): AsyncGenerator } else if (req.provider === "anthropic") { const client = anthropicClient(); - const system = req.messages.find((m) => m.role === "system")?.content; - const msgs = req.messages - .filter((m) => m.role !== "system") - .map((m) => ({ role: m.role === "assistant" ? "assistant" : "user", content: m.content })); + const system = getAnthropicSystemPrompt(req.messages); + const msgs = req.messages.filter((message) => message.role !== "system").map((message) => buildAnthropicConversationMessage(message)); const stream = await client.messages.create({ model: req.model, diff --git a/server/src/llm/types.ts b/server/src/llm/types.ts index bde093a..516c8c5 100644 --- a/server/src/llm/types.ts +++ b/server/src/llm/types.ts @@ -1,9 +1,31 @@ export type Provider = "openai" | "anthropic" | "xai"; +export type ChatImageAttachment = { + kind: "image"; + id: string; + filename: string; + mimeType: "image/png" | "image/jpeg"; + sizeBytes: number; + dataUrl: string; +}; + +export type ChatTextAttachment = { + kind: "text"; + id: string; + filename: string; + mimeType: string; + sizeBytes: number; + text: string; + truncated?: boolean; +}; + +export type ChatAttachment = ChatImageAttachment | ChatTextAttachment; + export type ChatMessage = { role: "system" | "user" | "assistant" | "tool"; content: string; name?: string; + attachments?: ChatAttachment[]; }; export type MultiplexRequest = { diff --git a/server/src/routes.ts b/server/src/routes.ts index ac41e86..1eb49cc 100644 --- a/server/src/routes.ts +++ b/server/src/routes.ts @@ -4,23 +4,33 @@ import type { FastifyInstance } from "fastify"; import { prisma } from "./db.js"; import { requireAdmin } from "./auth.js"; import { env } from "./env.js"; +import { buildComparableAttachments } from "./llm/message-content.js"; import { runMultiplex } from "./llm/multiplexer.js"; import { runMultiplexStream } from "./llm/streaming.js"; import { getModelCatalogSnapshot } from "./llm/model-catalog.js"; import { openaiClient } from "./llm/providers.js"; import { exaClient } from "./search/exa.js"; +import type { ChatAttachment } from "./llm/types.js"; type IncomingChatMessage = { role: "system" | "user" | "assistant" | "tool"; content: string; name?: string; + attachments?: ChatAttachment[]; }; function sameMessage( - a: { role: string; content: string; name?: string | null }, - b: { role: string; content: string; name?: string | null } + a: { role: string; content: string; name?: string | null; metadata?: unknown }, + b: { role: string; content: string; name?: string | null; attachments?: ChatAttachment[] } ) { - return a.role === b.role && a.content === b.content && (a.name ?? null) === (b.name ?? null); + const existingAttachments = JSON.stringify(buildComparableAttachments((a.metadata as Record | null)?.attachments ?? null)); + const incomingAttachments = JSON.stringify(b.attachments ?? []); + return ( + a.role === b.role && + a.content === b.content && + (a.name ?? null) === (b.name ?? null) && + existingAttachments === incomingAttachments + ); } function isToolCallLogMetadata(value: unknown) { @@ -60,10 +70,67 @@ async function storeNonAssistantMessages(chatId: string, messages: IncomingChatM role: m.role as any, content: m.content, name: m.name, + metadata: m.attachments?.length ? ({ attachments: m.attachments } as any) : undefined, })), }); } +const MAX_CHAT_ATTACHMENTS = 8; +const MAX_IMAGE_ATTACHMENT_BYTES = 6 * 1024 * 1024; +const MAX_TEXT_ATTACHMENT_CHARS = 200_000; +const MAX_IMAGE_DATA_URL_CHARS = 8_500_000; + +const ChatAttachmentSchema = z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("image"), + id: z.string().trim().min(1).max(128), + filename: z.string().trim().min(1).max(255), + mimeType: z.enum(["image/png", "image/jpeg"]), + sizeBytes: z.number().int().positive().max(MAX_IMAGE_ATTACHMENT_BYTES), + dataUrl: z + .string() + .max(MAX_IMAGE_DATA_URL_CHARS) + .regex(/^data:image\/(?:png|jpeg);base64,[a-z0-9+/=\s]+$/i, "Invalid image data URL"), + }), + z.object({ + kind: z.literal("text"), + id: z.string().trim().min(1).max(128), + filename: z.string().trim().min(1).max(255), + mimeType: z.string().trim().min(1).max(127), + sizeBytes: z.number().int().positive().max(8 * 1024 * 1024), + text: z.string().max(MAX_TEXT_ATTACHMENT_CHARS), + truncated: z.boolean().optional(), + }), +]); + +const CompletionMessageSchema = z + .object({ + role: z.enum(["system", "user", "assistant", "tool"]), + content: z.string(), + name: z.string().optional(), + attachments: z.array(ChatAttachmentSchema).max(MAX_CHAT_ATTACHMENTS).optional(), + }) + .superRefine((value, ctx) => { + if (value.attachments?.length && value.role === "tool") { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Tool messages cannot include attachments.", + path: ["attachments"], + }); + } + }); + +function mergeAttachmentsIntoMetadata(metadata: unknown, attachments?: ChatAttachment[]) { + if (!attachments?.length) return metadata as any; + if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) { + return { attachments }; + } + return { + ...(metadata as Record), + attachments, + }; +} + const SearchRunBody = z.object({ query: z.string().trim().min(1).optional(), title: z.string().trim().min(1).optional(), @@ -768,6 +835,7 @@ export async function registerRoutes(app: FastifyInstance) { content: z.string(), name: z.string().optional(), metadata: z.unknown().optional(), + attachments: z.array(ChatAttachmentSchema).max(MAX_CHAT_ATTACHMENTS).optional(), }); const { chatId } = Params.parse(req.params); @@ -779,7 +847,7 @@ export async function registerRoutes(app: FastifyInstance) { role: body.role as any, content: body.content, name: body.name, - metadata: body.metadata as any, + metadata: mergeAttachmentsIntoMetadata(body.metadata, body.attachments) as any, }, }); @@ -794,13 +862,7 @@ export async function registerRoutes(app: FastifyInstance) { chatId: z.string().optional(), provider: z.enum(["openai", "anthropic", "xai"]), model: z.string().min(1), - messages: z.array( - z.object({ - role: z.enum(["system", "user", "assistant", "tool"]), - content: z.string(), - name: z.string().optional(), - }) - ), + messages: z.array(CompletionMessageSchema), temperature: z.number().min(0).max(2).optional(), maxTokens: z.number().int().positive().optional(), }); @@ -834,13 +896,7 @@ export async function registerRoutes(app: FastifyInstance) { chatId: z.string().optional(), provider: z.enum(["openai", "anthropic", "xai"]), model: z.string().min(1), - messages: z.array( - z.object({ - role: z.enum(["system", "user", "assistant", "tool"]), - content: z.string(), - name: z.string().optional(), - }) - ), + messages: z.array(CompletionMessageSchema), temperature: z.number().min(0).max(2).optional(), maxTokens: z.number().int().positive().optional(), }); diff --git a/web/src/App.tsx b/web/src/App.tsx index 051d7df..3c03f58 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,9 +1,10 @@ import { useEffect, useMemo, useRef, useState } from "preact/hooks"; -import { Check, ChevronDown, Globe2, Menu, MessageSquare, Plus, Search, SendHorizontal, Trash2 } from "lucide-preact"; +import { Check, ChevronDown, Globe2, Menu, MessageSquare, Paperclip, Plus, Search, SendHorizontal, Trash2 } from "lucide-preact"; import { Button } from "@/components/ui/button"; import { Textarea } from "@/components/ui/textarea"; import { Separator } from "@/components/ui/separator"; import { AuthScreen } from "@/components/auth/auth-screen"; +import { ChatAttachmentList } from "@/components/chat/chat-attachment-list"; import { ChatMessagesPanel } from "@/components/chat/chat-messages-panel"; import { SearchResultsPanel } from "@/components/search/search-results-panel"; import { @@ -20,6 +21,8 @@ import { runCompletionStream, runSearchStream, suggestChatTitle, + getMessageAttachments, + type ChatAttachment, type ModelCatalogResponse, type Provider, type ChatDetail, @@ -102,6 +105,65 @@ const TRANSCRIPT_BOTTOM_GAP = 20; const REPLY_SCROLL_BUFFER_MIN = 288; const REPLY_SCROLL_BUFFER_MAX = 576; const REPLY_SCROLL_BUFFER_VIEWPORT_RATIO = 0.52; +const MAX_CHAT_ATTACHMENTS = 8; +const MAX_IMAGE_ATTACHMENT_BYTES = 6 * 1024 * 1024; +const MAX_TEXT_ATTACHMENT_BYTES = 8 * 1024 * 1024; +const MAX_TEXT_ATTACHMENT_CHARS = 200_000; +const CHAT_FILE_ACCEPT = + ".png,.jpg,.jpeg,.txt,.md,.markdown,.csv,.tsv,.json,.jsonl,.xml,.yaml,.yml,.html,.htm,.css,.js,.jsx,.ts,.tsx,.py,.rb,.java,.c,.cc,.cpp,.h,.hpp,.go,.rs,.sh,.sql,.log,.toml,.ini,.cfg,.conf,.swift,.kt,.m,.mm"; +const TEXT_ATTACHMENT_EXTENSIONS = new Set([ + ".txt", + ".md", + ".markdown", + ".csv", + ".tsv", + ".json", + ".jsonl", + ".xml", + ".yaml", + ".yml", + ".html", + ".htm", + ".css", + ".js", + ".jsx", + ".ts", + ".tsx", + ".py", + ".rb", + ".java", + ".c", + ".cc", + ".cpp", + ".h", + ".hpp", + ".go", + ".rs", + ".sh", + ".sql", + ".log", + ".toml", + ".ini", + ".cfg", + ".conf", + ".swift", + ".kt", + ".m", + ".mm", +]); +const TEXT_ATTACHMENT_MIME_TYPES = new Set([ + "application/json", + "application/ld+json", + "application/sql", + "application/toml", + "application/x-httpd-php", + "application/x-javascript", + "application/x-sh", + "application/xml", + "application/yaml", + "application/x-yaml", + "image/svg+xml", +]); function getModelOptions(catalog: ModelCatalogResponse["providers"], provider: Provider) { const providerModels = catalog[provider]?.models ?? []; @@ -117,6 +179,103 @@ function getReplyScrollBufferHeight() { ); } +function getFileExtension(filename: string) { + const index = filename.lastIndexOf("."); + return index >= 0 ? filename.slice(index).toLowerCase() : ""; +} + +function createAttachmentId() { + if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") { + return crypto.randomUUID(); + } + return `att-${Date.now()}-${Math.random().toString(36).slice(2, 10)}`; +} + +function inferImageMimeType(file: File) { + if (file.type === "image/png" || file.type === "image/jpeg") return file.type; + const extension = getFileExtension(file.name); + if (extension === ".png") return "image/png"; + if (extension === ".jpg" || extension === ".jpeg") return "image/jpeg"; + return null; +} + +function isTextLikeFile(file: File) { + const mimeType = file.type.toLowerCase(); + if (mimeType.startsWith("text/")) return true; + if (TEXT_ATTACHMENT_MIME_TYPES.has(mimeType)) return true; + return TEXT_ATTACHMENT_EXTENSIONS.has(getFileExtension(file.name)); +} + +function arrayBufferToBase64(buffer: ArrayBuffer) { + const bytes = new Uint8Array(buffer); + const chunkSize = 0x8000; + let binary = ""; + for (let index = 0; index < bytes.length; index += chunkSize) { + const chunk = bytes.subarray(index, index + chunkSize); + binary += String.fromCharCode(...chunk); + } + return btoa(binary); +} + +async function buildChatAttachment(file: File): Promise { + const imageMimeType = inferImageMimeType(file); + if (imageMimeType) { + if (file.size > MAX_IMAGE_ATTACHMENT_BYTES) { + throw new Error(`Image '${file.name}' exceeds the 6 MB upload limit.`); + } + const base64 = arrayBufferToBase64(await file.arrayBuffer()); + return { + kind: "image", + id: createAttachmentId(), + filename: file.name, + mimeType: imageMimeType, + sizeBytes: file.size, + dataUrl: `data:${imageMimeType};base64,${base64}`, + }; + } + + if (!isTextLikeFile(file)) { + throw new Error(`Unsupported file type for '${file.name}'. Use PNG/JPEG images or text-based files.`); + } + + if (file.size > MAX_TEXT_ATTACHMENT_BYTES) { + throw new Error(`Text file '${file.name}' exceeds the 8 MB upload limit.`); + } + + const normalizedText = (await file.text()).replace(/\r\n/g, "\n").replace(/\u0000/g, ""); + const truncated = normalizedText.length > MAX_TEXT_ATTACHMENT_CHARS; + return { + kind: "text", + id: createAttachmentId(), + filename: file.name, + mimeType: file.type || "text/plain", + sizeBytes: file.size, + text: truncated ? normalizedText.slice(0, MAX_TEXT_ATTACHMENT_CHARS) : normalizedText, + truncated, + }; +} + +function buildAttachmentSummary(attachments: ChatAttachment[]) { + if (!attachments.length) return ""; + const filenames = attachments.map((attachment) => attachment.filename).join(", "); + return attachments.length === 1 ? filenames : `Attached: ${filenames}`; +} + +function getFilesFromDataTransfer(dataTransfer: DataTransfer | null) { + if (!dataTransfer) return []; + const fromItems = Array.from(dataTransfer.items ?? []) + .filter((item) => item.kind === "file") + .map((item) => item.getAsFile()) + .filter((file): file is File => file instanceof File); + if (fromItems.length) return fromItems; + return Array.from(dataTransfer.files ?? []); +} + +function hasFileTransfer(dataTransfer: DataTransfer | null) { + if (!dataTransfer) return false; + return Array.from(dataTransfer.types ?? []).includes("Files") || getFilesFromDataTransfer(dataTransfer).length > 0; +} + function loadStoredModelPreferences() { if (typeof window === "undefined") return EMPTY_MODEL_PREFERENCES; try { @@ -347,8 +506,12 @@ function ModelCombobox({ options, value, onChange, disabled = false }: ModelComb function getChatTitle(chat: Pick, messages?: ChatDetail["messages"]) { if (chat.title?.trim()) return chat.title.trim(); - const firstUserMessage = messages?.find((m) => m.role === "user")?.content.trim(); - if (firstUserMessage) return firstUserMessage.slice(0, 48); + const firstUserMessage = messages?.find((message) => message.role === "user"); + const firstUserText = firstUserMessage?.content.trim(); + if (firstUserText) return firstUserText.slice(0, 48); + const firstUserAttachments = firstUserMessage ? getMessageAttachments(firstUserMessage.metadata) : []; + const attachmentSummary = buildAttachmentSummary(firstUserAttachments); + if (attachmentSummary) return attachmentSummary.slice(0, 48); return "New chat"; } @@ -448,6 +611,8 @@ export default function App() { const [isStartingSearchChat, setIsStartingSearchChat] = useState(false); const [pendingChatState, setPendingChatState] = useState<{ chatId: string | null; messages: Message[] } | null>(null); const [composer, setComposer] = useState(""); + const [pendingAttachments, setPendingAttachments] = useState([]); + const [isComposerDropActive, setIsComposerDropActive] = useState(false); const [provider, setProvider] = useState("openai"); const [modelCatalog, setModelCatalog] = useState(EMPTY_MODEL_CATALOG); const [providerModelPreferences, setProviderModelPreferences] = useState(() => loadStoredModelPreferences()); @@ -460,6 +625,9 @@ export default function App() { const transcriptContainerRef = useRef(null); const transcriptEndRef = useRef(null); const contextMenuRef = useRef(null); + const fileInputRef = useRef(null); + const dragDepthRef = useRef(0); + const pendingAttachmentsRef = useRef([]); const selectedItemRef = useRef(null); const pendingTitleGenerationRef = useRef>(new Set()); const searchRunAbortRef = useRef(null); @@ -518,6 +686,10 @@ export default function App() { textarea.style.height = `${textarea.scrollHeight}px`; }, [composer]); + useEffect(() => { + pendingAttachmentsRef.current = pendingAttachments; + }, [pendingAttachments]); + const sidebarItems = useMemo(() => buildSidebarItems(chats, searches), [chats, searches]); const filteredSidebarItems = useMemo(() => { const query = sidebarQuery.trim().toLowerCase(); @@ -540,6 +712,7 @@ export default function App() { setDraftKind(null); setPendingChatState(null); setComposer(""); + setPendingAttachments([]); setError(null); }; @@ -767,6 +940,16 @@ export default function App() { const isSearchMode = draftKind ? draftKind === "search" : selectedItem?.kind === "search"; const isSearchRunning = isSending && isSearchMode; const isSendingActiveChat = isChatReplyStreamingInView; + + useEffect(() => { + if (isSearchMode && pendingAttachments.length) { + setPendingAttachments([]); + } + if (isSearchMode) { + dragDepthRef.current = 0; + setIsComposerDropActive(false); + } + }, [isSearchMode, pendingAttachments.length]); const displayMessages = useMemo(() => { if (!pendingChatState) return messages.filter(isDisplayableMessage); if (pendingChatState.chatId) { @@ -837,6 +1020,7 @@ export default function App() { setSelectedItem(null); setSelectedChat(null); setSelectedSearch(null); + setPendingAttachments([]); setIsMobileSidebarOpen(false); }; @@ -847,6 +1031,7 @@ export default function App() { setSelectedItem(null); setSelectedChat(null); setSelectedSearch(null); + setPendingAttachments([]); setIsMobileSidebarOpen(false); }; @@ -899,7 +1084,88 @@ export default function App() { }; }, [contextMenu]); - const handleSendChat = async (content: string) => { + const handleOpenAttachmentPicker = () => { + fileInputRef.current?.click(); + }; + + const handleRemovePendingAttachment = (attachmentId: string) => { + setPendingAttachments((current) => current.filter((attachment) => attachment.id !== attachmentId)); + }; + + const appendPendingAttachments = async (files: File[]) => { + if (!files.length) return; + if (isSearchMode) { + setError("Attachments are only available in chat mode."); + return; + } + + setError(null); + + try { + const attachments = await Promise.all(files.map((file) => buildChatAttachment(file))); + if (pendingAttachmentsRef.current.length + attachments.length > MAX_CHAT_ATTACHMENTS) { + throw new Error(`You can attach up to ${MAX_CHAT_ATTACHMENTS} files per message.`); + } + setPendingAttachments((current) => current.concat(attachments)); + focusComposer(); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + setError(message); + } + }; + + const handleFileSelection = async (event: Event) => { + const input = event.currentTarget as HTMLInputElement; + const files = Array.from(input.files ?? []); + input.value = ""; + await appendPendingAttachments(files); + }; + + const handleComposerPaste = async (event: ClipboardEvent) => { + const files = getFilesFromDataTransfer(event.clipboardData); + if (!files.length) return; + event.preventDefault(); + await appendPendingAttachments(files); + }; + + const handleComposerDragEnter = (event: DragEvent) => { + if (!hasFileTransfer(event.dataTransfer)) return; + event.preventDefault(); + if (isSearchMode) return; + dragDepthRef.current += 1; + setIsComposerDropActive(true); + }; + + const handleComposerDragOver = (event: DragEvent) => { + if (!hasFileTransfer(event.dataTransfer)) return; + event.preventDefault(); + if (event.dataTransfer) { + event.dataTransfer.dropEffect = isSearchMode ? "none" : "copy"; + } + if (!isSearchMode) { + setIsComposerDropActive(true); + } + }; + + const handleComposerDragLeave = (event: DragEvent) => { + if (!hasFileTransfer(event.dataTransfer)) return; + event.preventDefault(); + if (isSearchMode) return; + dragDepthRef.current = Math.max(0, dragDepthRef.current - 1); + if (dragDepthRef.current === 0) { + setIsComposerDropActive(false); + } + }; + + const handleComposerDrop = async (event: DragEvent) => { + if (!hasFileTransfer(event.dataTransfer)) return; + event.preventDefault(); + dragDepthRef.current = 0; + setIsComposerDropActive(false); + await appendPendingAttachments(getFilesFromDataTransfer(event.dataTransfer)); + }; + + const handleSendChat = async (content: string, attachments: ChatAttachment[]) => { pendingReplyScrollRef.current = true; expandTranscriptTailSpacer(getReplyScrollBufferHeight()); @@ -909,7 +1175,7 @@ export default function App() { role: "user", content, name: null, - metadata: null, + metadata: attachments.length ? { attachments } : null, }; const optimisticAssistantMessage: Message = { @@ -965,13 +1231,15 @@ export default function App() { ...baseChat.messages .filter((message) => !isToolCallLogMessage(message)) .map((message) => ({ - role: message.role, - content: message.content, - ...(message.name ? { name: message.name } : {}), + role: message.role, + content: message.content, + ...(message.name ? { name: message.name } : {}), + ...(getMessageAttachments(message.metadata).length ? { attachments: getMessageAttachments(message.metadata) } : {}), })), { role: "user", content, + ...(attachments.length ? { attachments } : {}), }, ]; @@ -984,7 +1252,8 @@ export default function App() { const hasExistingTitle = Boolean(selectedChat?.id === chatId ? selectedChat.title?.trim() : chatSummary?.title?.trim()); if (!hasExistingTitle && !pendingTitleGenerationRef.current.has(chatId)) { pendingTitleGenerationRef.current.add(chatId); - void suggestChatTitle({ chatId, content }) + const titleSeed = content || buildAttachmentSummary(attachments) || "Uploaded files"; + void suggestChatTitle({ chatId, content: titleSeed }) .then((updatedChat) => { setChats((current) => current.map((chat) => { @@ -1232,6 +1501,7 @@ export default function App() { setDraftKind(null); setPendingChatState(null); setComposer(""); + setPendingAttachments([]); setChats((current) => { const withoutExisting = current.filter((existing) => existing.id !== chat.id); return [chat, ...withoutExisting]; @@ -1265,9 +1535,15 @@ export default function App() { const handleSend = async () => { const content = composer.trim(); - if (!content || isSending) return; + const attachments = pendingAttachments; + if ((!content && !attachments.length) || isSending) return; + if (isSearchMode && attachments.length) { + setError("Attachments are only available in chat mode."); + return; + } setComposer(""); + setPendingAttachments([]); setError(null); setIsSending(true); @@ -1275,7 +1551,7 @@ export default function App() { if (isSearchMode) { await handleSendSearch(content); } else { - await handleSendChat(content); + await handleSendChat(content, attachments); } } catch (err) { const message = err instanceof Error ? err.message : String(err); @@ -1286,6 +1562,8 @@ export default function App() { } if (!isSearchMode) { + setComposer(content); + setPendingAttachments(attachments); setPendingChatState(null); } @@ -1519,7 +1797,42 @@ export default function App() {