import type { FastifyBaseLogger } from "fastify"; import { env } from "../env.js"; import { anthropicClient, hermesAgentClient, isHermesAgentConfigured, openaiClient, xaiClient } from "./providers.js"; import type { Provider } from "./types.js"; export type ProviderModelSnapshot = { models: string[]; loadedAt: string | null; error: string | null; }; export type ModelCatalogSnapshot = Partial>; const baseProviders: Provider[] = ["openai", "anthropic", "xai"]; const MODEL_FETCH_TIMEOUT_MS = 15000; const MODEL_CATALOG_REFRESH_INTERVAL_MS = 24 * 60 * 60 * 1000; const modelCatalog: ModelCatalogSnapshot = { openai: { models: [], loadedAt: null, error: null }, anthropic: { models: [], loadedAt: null, error: null }, xai: { models: [], loadedAt: null, error: null }, }; let catalogRefreshPromise: Promise | null = null; function getCatalogProviders(): Provider[] { return isHermesAgentConfigured() ? [...baseProviders, "hermes-agent"] : baseProviders; } function uniqSorted(models: string[]) { return [...new Set(models.map((value) => value.trim()).filter(Boolean))].sort((a, b) => a.localeCompare(b)); } function isLikelyOpenAIResponsesModel(model: string) { const id = model.toLowerCase(); if (id.includes("embedding") || id.includes("moderation")) return false; if (id.includes("audio") || id.includes("realtime") || id.includes("transcribe") || id.includes("tts")) return false; if (id.includes("image") || id.includes("dall-e") || id.includes("sora")) return false; if (id.includes("search") || id.includes("computer-use")) return false; return /^(gpt-|o\d|chatgpt-)/.test(id); } async function withTimeout(promise: Promise, timeoutMs: number, label: string) { let timeoutId: NodeJS.Timeout | null = null; try { return await Promise.race([ promise, new Promise((_, reject) => { timeoutId = setTimeout(() => { reject(new Error(`${label} timed out after ${timeoutMs}ms`)); }, timeoutMs); }), ]); } finally { if (timeoutId) clearTimeout(timeoutId); } } async function fetchProviderModels(provider: Provider) { if (provider === "openai") { const page = await openaiClient().models.list(); return uniqSorted(page.data.map((model) => model.id).filter(isLikelyOpenAIResponsesModel)); } if (provider === "anthropic") { const page = await anthropicClient().models.list({ limit: 200 }); return uniqSorted(page.data.map((model) => model.id)); } if (provider === "xai") { const page = await xaiClient().models.list(); return uniqSorted(page.data.map((model) => model.id)); } const page = await hermesAgentClient().models.list(); const models = page.data.map((model) => model.id); if (env.HERMES_AGENT_MODEL) models.push(env.HERMES_AGENT_MODEL); return uniqSorted(models); } async function refreshProviderModels(provider: Provider, logger?: FastifyBaseLogger) { try { const models = await withTimeout(fetchProviderModels(provider), MODEL_FETCH_TIMEOUT_MS, `${provider} model fetch`); modelCatalog[provider] = { models, loadedAt: new Date().toISOString(), error: null, }; logger?.info({ provider, modelCount: models.length }, "model catalog loaded"); } catch (err: any) { const message = err?.message ?? String(err); const previous = modelCatalog[provider]; const fallbackModels = provider === "hermes-agent" && env.HERMES_AGENT_MODEL ? [env.HERMES_AGENT_MODEL] : []; modelCatalog[provider] = { models: previous?.models.length ? previous.models : fallbackModels, loadedAt: previous?.loadedAt ?? null, error: message, }; logger?.warn({ provider, err: message }, "failed to load provider model catalog"); } } export async function refreshModelCatalog(logger?: FastifyBaseLogger) { if (catalogRefreshPromise) return catalogRefreshPromise; catalogRefreshPromise = Promise.all(getCatalogProviders().map((provider) => refreshProviderModels(provider, logger))) .then(() => undefined) .finally(() => { catalogRefreshPromise = null; }); return catalogRefreshPromise; } export async function warmModelCatalog(logger?: FastifyBaseLogger) { await refreshModelCatalog(logger); } export function startModelCatalogRefreshLoop(logger?: FastifyBaseLogger) { const timer = setInterval(() => { void refreshModelCatalog(logger); }, MODEL_CATALOG_REFRESH_INTERVAL_MS); timer.unref?.(); return () => { clearInterval(timer); }; } export function getModelCatalogSnapshot(): ModelCatalogSnapshot { const snapshot: ModelCatalogSnapshot = {}; for (const provider of getCatalogProviders()) { const entry = modelCatalog[provider] ?? { models: [], loadedAt: null, error: null }; snapshot[provider] = { models: [...entry.models], loadedAt: entry.loadedAt, error: entry.error, }; } return snapshot; }