From cd449a30fa5beae366c3ebae80699ce37965715d Mon Sep 17 00:00:00 2001 From: James Magahern Date: Sat, 14 Feb 2026 21:00:30 -0800 Subject: [PATCH] model catalog --- server/src/index.ts | 2 + server/src/llm/model-catalog.ts | 99 +++++++++++++++ server/src/routes.ts | 6 + web/src/App.tsx | 215 ++++++++++++++++++++++++++++++-- web/src/lib/api.ts | 18 ++- 5 files changed, 327 insertions(+), 13 deletions(-) create mode 100644 server/src/llm/model-catalog.ts diff --git a/server/src/index.ts b/server/src/index.ts index 7b40dab..45662f4 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -5,6 +5,7 @@ import swaggerUI from "@fastify/swagger-ui"; import sensible from "@fastify/sensible"; import { env } from "./env.js"; import { ensureDatabaseReady } from "./db-init.js"; +import { warmModelCatalog } from "./llm/model-catalog.js"; import { registerRoutes } from "./routes.js"; const app = Fastify({ @@ -18,6 +19,7 @@ const app = Fastify({ }); await ensureDatabaseReady(app.log); +await warmModelCatalog(app.log); await app.register(cors, { origin: true, diff --git a/server/src/llm/model-catalog.ts b/server/src/llm/model-catalog.ts new file mode 100644 index 0000000..54a5673 --- /dev/null +++ b/server/src/llm/model-catalog.ts @@ -0,0 +1,99 @@ +import type { FastifyBaseLogger } from "fastify"; +import { anthropicClient, openaiClient, xaiClient } from "./providers.js"; +import type { Provider } from "./types.js"; + +export type ProviderModelSnapshot = { + models: string[]; + loadedAt: string | null; + error: string | null; +}; + +export type ModelCatalogSnapshot = Record; + +const providers: Provider[] = ["openai", "anthropic", "xai"]; +const MODEL_FETCH_TIMEOUT_MS = 15000; + +const modelCatalog: ModelCatalogSnapshot = { + openai: { models: [], loadedAt: null, error: null }, + anthropic: { models: [], loadedAt: null, error: null }, + xai: { models: [], loadedAt: null, error: null }, +}; + +function uniqSorted(models: string[]) { + return [...new Set(models.map((value) => value.trim()).filter(Boolean))].sort((a, b) => a.localeCompare(b)); +} + +async function withTimeout(promise: Promise, timeoutMs: number, label: string) { + let timeoutId: NodeJS.Timeout | null = null; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timeoutId = setTimeout(() => { + reject(new Error(`${label} timed out after ${timeoutMs}ms`)); + }, timeoutMs); + }), + ]); + } finally { + if (timeoutId) clearTimeout(timeoutId); + } +} + +async function fetchProviderModels(provider: Provider) { + if (provider === "openai") { + const page = await openaiClient().models.list(); + return uniqSorted(page.data.map((model) => model.id)); + } + + if (provider === "anthropic") { + const page = await anthropicClient().models.list({ limit: 200 }); + return uniqSorted(page.data.map((model) => model.id)); + } + + const page = await xaiClient().models.list(); + return uniqSorted(page.data.map((model) => model.id)); +} + +async function refreshProviderModels(provider: Provider, logger?: FastifyBaseLogger) { + try { + const models = await withTimeout(fetchProviderModels(provider), MODEL_FETCH_TIMEOUT_MS, `${provider} model fetch`); + modelCatalog[provider] = { + models, + loadedAt: new Date().toISOString(), + error: null, + }; + logger?.info({ provider, modelCount: models.length }, "model catalog loaded"); + } catch (err: any) { + const message = err?.message ?? String(err); + modelCatalog[provider] = { + models: [], + loadedAt: new Date().toISOString(), + error: message, + }; + logger?.warn({ provider, err: message }, "failed to load provider model catalog"); + } +} + +export async function warmModelCatalog(logger?: FastifyBaseLogger) { + await Promise.all(providers.map((provider) => refreshProviderModels(provider, logger))); +} + +export function getModelCatalogSnapshot(): ModelCatalogSnapshot { + return { + openai: { + models: [...modelCatalog.openai.models], + loadedAt: modelCatalog.openai.loadedAt, + error: modelCatalog.openai.error, + }, + anthropic: { + models: [...modelCatalog.anthropic.models], + loadedAt: modelCatalog.anthropic.loadedAt, + error: modelCatalog.anthropic.error, + }, + xai: { + models: [...modelCatalog.xai.models], + loadedAt: modelCatalog.xai.loadedAt, + error: modelCatalog.xai.error, + }, + }; +} diff --git a/server/src/routes.ts b/server/src/routes.ts index 0976e4c..62497e9 100644 --- a/server/src/routes.ts +++ b/server/src/routes.ts @@ -6,6 +6,7 @@ import { requireAdmin } from "./auth.js"; import { env } from "./env.js"; import { runMultiplex } from "./llm/multiplexer.js"; import { runMultiplexStream } from "./llm/streaming.js"; +import { getModelCatalogSnapshot } from "./llm/model-catalog.js"; import { exaClient } from "./search/exa.js"; type IncomingChatMessage = { @@ -135,6 +136,11 @@ export async function registerRoutes(app: FastifyInstance) { return { authenticated: true, mode: env.ADMIN_TOKEN ? "token" : "open" }; }); + app.get("/v1/models", async (req) => { + requireAdmin(req); + return { providers: getModelCatalogSnapshot() }; + }); + app.get("/v1/chats", async (req) => { requireAdmin(req); const chats = await prisma.chat.findMany({ diff --git a/web/src/App.tsx b/web/src/App.tsx index c996d81..79fa5f8 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -1,7 +1,6 @@ import { useEffect, useMemo, useRef, useState } from "preact/hooks"; -import { Globe2, LogOut, MessageSquare, Plus, Search, SendHorizontal, Trash2 } from "lucide-preact"; +import { Check, ChevronDown, Globe2, LogOut, MessageSquare, Plus, Search, SendHorizontal, Trash2 } from "lucide-preact"; import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; import { Textarea } from "@/components/ui/textarea"; import { Separator } from "@/components/ui/separator"; import { AuthScreen } from "@/components/auth/auth-screen"; @@ -13,11 +12,14 @@ import { deleteChat, deleteSearch, getChat, + listModels, getSearch, listChats, listSearches, runCompletion, runSearchStream, + type ModelCatalogResponse, + type Provider, type ChatDetail, type ChatSummary, type CompletionRequestMessage, @@ -28,7 +30,6 @@ import { import { useSessionAuth } from "@/hooks/use-session-auth"; import { cn } from "@/lib/utils"; -type Provider = "openai" | "anthropic" | "xai"; type SidebarSelection = { kind: "chat" | "search"; id: string }; type DraftSelectionKind = "chat" | "search"; type SidebarItem = SidebarSelection & { @@ -42,12 +43,153 @@ type ContextMenuState = { y: number; }; -const PROVIDER_DEFAULT_MODELS: Record = { - openai: "gpt-4.1-mini", - anthropic: "claude-3-5-sonnet-latest", - xai: "grok-3-mini", +const PROVIDER_FALLBACK_MODELS: Record = { + openai: ["gpt-4.1-mini"], + anthropic: ["claude-3-5-sonnet-latest"], + xai: ["grok-3-mini"], }; +const EMPTY_MODEL_CATALOG: ModelCatalogResponse["providers"] = { + openai: { models: [], loadedAt: null, error: null }, + anthropic: { models: [], loadedAt: null, error: null }, + xai: { models: [], loadedAt: null, error: null }, +}; + +const MODEL_PREFERENCES_STORAGE_KEY = "sybil:modelPreferencesByProvider"; + +type ProviderModelPreferences = Record; + +const EMPTY_MODEL_PREFERENCES: ProviderModelPreferences = { + openai: null, + anthropic: null, + xai: null, +}; + +function getModelOptions(catalog: ModelCatalogResponse["providers"], provider: Provider) { + const providerModels = catalog[provider]?.models ?? []; + if (providerModels.length) return providerModels; + return PROVIDER_FALLBACK_MODELS[provider]; +} + +function loadStoredModelPreferences() { + if (typeof window === "undefined") return EMPTY_MODEL_PREFERENCES; + try { + const raw = window.localStorage.getItem(MODEL_PREFERENCES_STORAGE_KEY); + if (!raw) return EMPTY_MODEL_PREFERENCES; + const parsed = JSON.parse(raw) as Partial>; + return { + openai: typeof parsed.openai === "string" && parsed.openai.trim() ? parsed.openai.trim() : null, + anthropic: typeof parsed.anthropic === "string" && parsed.anthropic.trim() ? parsed.anthropic.trim() : null, + xai: typeof parsed.xai === "string" && parsed.xai.trim() ? parsed.xai.trim() : null, + }; + } catch { + return EMPTY_MODEL_PREFERENCES; + } +} + +function pickProviderModel(options: string[], preferred: string | null, fallback: string | null = null) { + if (preferred && options.includes(preferred)) return preferred; + if (fallback && options.includes(fallback)) return fallback; + return options[0] ?? ""; +} + +type ModelComboboxProps = { + options: string[]; + value: string; + onChange: (value: string) => void; + disabled?: boolean; +}; + +function ModelCombobox({ options, value, onChange, disabled = false }: ModelComboboxProps) { + const [open, setOpen] = useState(false); + const [query, setQuery] = useState(""); + const rootRef = useRef(null); + const inputRef = useRef(null); + + const filteredOptions = useMemo(() => { + const needle = query.trim().toLowerCase(); + if (!needle) return options; + return options.filter((option) => option.toLowerCase().includes(needle)); + }, [options, query]); + + useEffect(() => { + if (!open) return; + inputRef.current?.focus(); + }, [open]); + + useEffect(() => { + if (!open) return; + + const handlePointerDown = (event: PointerEvent) => { + if (rootRef.current?.contains(event.target as Node)) return; + setOpen(false); + setQuery(""); + }; + + const handleKeyDown = (event: KeyboardEvent) => { + if (event.key !== "Escape") return; + setOpen(false); + setQuery(""); + }; + + window.addEventListener("pointerdown", handlePointerDown); + window.addEventListener("keydown", handleKeyDown); + return () => { + window.removeEventListener("pointerdown", handlePointerDown); + window.removeEventListener("keydown", handleKeyDown); + }; + }, [open]); + + return ( +
+ + {open ? ( +
+ setQuery(event.currentTarget.value)} + className="mb-1 h-8 w-full rounded-sm border border-input bg-background px-2 text-sm outline-none" + placeholder="Filter models" + /> +
+ {filteredOptions.length ? ( + filteredOptions.map((option) => ( + + )) + ) : ( +

No models found

+ )} +
+
+ ) : null} +
+ ); +} + function getChatTitle(chat: Pick, messages?: ChatDetail["messages"]) { if (chat.title?.trim()) return chat.title.trim(); const firstUserMessage = messages?.find((m) => m.role === "user")?.content.trim(); @@ -117,7 +259,12 @@ export default function App() { const [pendingChatState, setPendingChatState] = useState<{ chatId: string | null; messages: Message[] } | null>(null); const [composer, setComposer] = useState(""); const [provider, setProvider] = useState("openai"); - const [model, setModel] = useState(PROVIDER_DEFAULT_MODELS.openai); + const [modelCatalog, setModelCatalog] = useState(EMPTY_MODEL_CATALOG); + const [providerModelPreferences, setProviderModelPreferences] = useState(() => loadStoredModelPreferences()); + const [model, setModel] = useState(() => { + const stored = loadStoredModelPreferences(); + return stored.openai ?? PROVIDER_FALLBACK_MODELS.openai[0]; + }); const [error, setError] = useState(null); const transcriptEndRef = useRef(null); const contextMenuRef = useRef(null); @@ -179,6 +326,20 @@ export default function App() { } }; + const refreshModels = async () => { + try { + const data = await listModels(); + setModelCatalog(data.providers); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + if (message.includes("bearer token")) { + handleAuthFailure(message); + } else { + setError(message); + } + } + }; + const refreshChat = async (chatId: string) => { setIsLoadingSelection(true); try { @@ -217,9 +378,22 @@ export default function App() { useEffect(() => { if (!isAuthenticated) return; - void refreshCollections(); + void Promise.all([refreshCollections(), refreshModels()]); }, [isAuthenticated]); + const providerModelOptions = useMemo(() => getModelOptions(modelCatalog, provider), [modelCatalog, provider]); + + useEffect(() => { + setModel((current) => { + return pickProviderModel(providerModelOptions, providerModelPreferences[provider], current); + }); + }, [provider, providerModelOptions, providerModelPreferences]); + + useEffect(() => { + if (typeof window === "undefined") return; + window.localStorage.setItem(MODEL_PREFERENCES_STORAGE_KEY, JSON.stringify(providerModelPreferences)); + }, [providerModelPreferences]); + const selectedKey = selectedItem ? `${selectedItem.kind}:${selectedItem.id}` : null; useEffect(() => { @@ -419,10 +593,15 @@ export default function App() { }, ]; + const selectedModel = model.trim(); + if (!selectedModel) { + throw new Error("No model available for selected provider"); + } + await runCompletion({ chatId, provider, - model: model.trim(), + model: selectedModel, messages: requestMessages, }); @@ -690,7 +869,8 @@ export default function App() { onChange={(event) => { const nextProvider = event.currentTarget.value as Provider; setProvider(nextProvider); - setModel(PROVIDER_DEFAULT_MODELS[nextProvider]); + const options = getModelOptions(modelCatalog, nextProvider); + setModel((current) => pickProviderModel(options, providerModelPreferences[nextProvider], current)); }} disabled={isSending} > @@ -698,7 +878,18 @@ export default function App() { - setModel(event.currentTarget.value)} placeholder="Model" disabled={isSending} /> + { + setModel(nextModel); + setProviderModelPreferences((current) => ({ + ...current, + [provider]: nextModel, + })); + }} + /> ) : (
diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index ccafd32..3b59a54 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -83,6 +83,18 @@ export type CompletionRequestMessage = { name?: string; }; +export type Provider = "openai" | "anthropic" | "xai"; + +export type ProviderModelInfo = { + models: string[]; + loadedAt: string | null; + error: string | null; +}; + +export type ModelCatalogResponse = { + providers: Record; +}; + type CompletionResponse = { chatId: string | null; message: { @@ -142,6 +154,10 @@ export async function verifySession() { return api<{ authenticated: true; mode: "open" | "token" }>("/v1/auth/session"); } +export async function listModels() { + return api("/v1/models"); +} + export async function createChat(title?: string) { const data = await api<{ chat: ChatSummary }>("/v1/chats", { method: "POST", @@ -296,7 +312,7 @@ export async function runSearchStream( export async function runCompletion(body: { chatId: string; - provider: "openai" | "anthropic" | "xai"; + provider: Provider; model: string; messages: CompletionRequestMessage[]; }) {