100 lines
3.1 KiB
TypeScript
100 lines
3.1 KiB
TypeScript
import type { FastifyBaseLogger } from "fastify";
|
|
import { anthropicClient, openaiClient, xaiClient } from "./providers.js";
|
|
import type { Provider } from "./types.js";
|
|
|
|
export type ProviderModelSnapshot = {
|
|
models: string[];
|
|
loadedAt: string | null;
|
|
error: string | null;
|
|
};
|
|
|
|
export type ModelCatalogSnapshot = Record<Provider, ProviderModelSnapshot>;
|
|
|
|
const providers: Provider[] = ["openai", "anthropic", "xai"];
|
|
const MODEL_FETCH_TIMEOUT_MS = 15000;
|
|
|
|
const modelCatalog: ModelCatalogSnapshot = {
|
|
openai: { models: [], loadedAt: null, error: null },
|
|
anthropic: { models: [], loadedAt: null, error: null },
|
|
xai: { models: [], loadedAt: null, error: null },
|
|
};
|
|
|
|
function uniqSorted(models: string[]) {
|
|
return [...new Set(models.map((value) => value.trim()).filter(Boolean))].sort((a, b) => a.localeCompare(b));
|
|
}
|
|
|
|
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string) {
|
|
let timeoutId: NodeJS.Timeout | null = null;
|
|
try {
|
|
return await Promise.race([
|
|
promise,
|
|
new Promise<T>((_, reject) => {
|
|
timeoutId = setTimeout(() => {
|
|
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
|
|
}, timeoutMs);
|
|
}),
|
|
]);
|
|
} finally {
|
|
if (timeoutId) clearTimeout(timeoutId);
|
|
}
|
|
}
|
|
|
|
async function fetchProviderModels(provider: Provider) {
|
|
if (provider === "openai") {
|
|
const page = await openaiClient().models.list();
|
|
return uniqSorted(page.data.map((model) => model.id));
|
|
}
|
|
|
|
if (provider === "anthropic") {
|
|
const page = await anthropicClient().models.list({ limit: 200 });
|
|
return uniqSorted(page.data.map((model) => model.id));
|
|
}
|
|
|
|
const page = await xaiClient().models.list();
|
|
return uniqSorted(page.data.map((model) => model.id));
|
|
}
|
|
|
|
async function refreshProviderModels(provider: Provider, logger?: FastifyBaseLogger) {
|
|
try {
|
|
const models = await withTimeout(fetchProviderModels(provider), MODEL_FETCH_TIMEOUT_MS, `${provider} model fetch`);
|
|
modelCatalog[provider] = {
|
|
models,
|
|
loadedAt: new Date().toISOString(),
|
|
error: null,
|
|
};
|
|
logger?.info({ provider, modelCount: models.length }, "model catalog loaded");
|
|
} catch (err: any) {
|
|
const message = err?.message ?? String(err);
|
|
modelCatalog[provider] = {
|
|
models: [],
|
|
loadedAt: new Date().toISOString(),
|
|
error: message,
|
|
};
|
|
logger?.warn({ provider, err: message }, "failed to load provider model catalog");
|
|
}
|
|
}
|
|
|
|
export async function warmModelCatalog(logger?: FastifyBaseLogger) {
|
|
await Promise.all(providers.map((provider) => refreshProviderModels(provider, logger)));
|
|
}
|
|
|
|
export function getModelCatalogSnapshot(): ModelCatalogSnapshot {
|
|
return {
|
|
openai: {
|
|
models: [...modelCatalog.openai.models],
|
|
loadedAt: modelCatalog.openai.loadedAt,
|
|
error: modelCatalog.openai.error,
|
|
},
|
|
anthropic: {
|
|
models: [...modelCatalog.anthropic.models],
|
|
loadedAt: modelCatalog.anthropic.loadedAt,
|
|
error: modelCatalog.anthropic.error,
|
|
},
|
|
xai: {
|
|
models: [...modelCatalog.xai.models],
|
|
loadedAt: modelCatalog.xai.loadedAt,
|
|
error: modelCatalog.xai.error,
|
|
},
|
|
};
|
|
}
|