101 lines
3.0 KiB
TypeScript
101 lines
3.0 KiB
TypeScript
import type { FastifyBaseLogger } from "fastify";
|
|
import {
|
|
fetchProviderCatalogModels,
|
|
getProviderCatalogFallbackModels,
|
|
listModelCatalogProviders,
|
|
} from "./provider-adapters.js";
|
|
import type { Provider } from "./types.js";
|
|
|
|
export type ProviderModelSnapshot = {
|
|
models: string[];
|
|
loadedAt: string | null;
|
|
error: string | null;
|
|
};
|
|
|
|
export type ModelCatalogSnapshot = Partial<Record<Provider, ProviderModelSnapshot>>;
|
|
|
|
const MODEL_FETCH_TIMEOUT_MS = 15000;
|
|
const MODEL_CATALOG_REFRESH_INTERVAL_MS = 24 * 60 * 60 * 1000;
|
|
|
|
const modelCatalog: ModelCatalogSnapshot = {};
|
|
|
|
let catalogRefreshPromise: Promise<void> | null = null;
|
|
|
|
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string) {
|
|
let timeoutId: NodeJS.Timeout | null = null;
|
|
try {
|
|
return await Promise.race([
|
|
promise,
|
|
new Promise<T>((_, reject) => {
|
|
timeoutId = setTimeout(() => {
|
|
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
|
|
}, timeoutMs);
|
|
}),
|
|
]);
|
|
} finally {
|
|
if (timeoutId) clearTimeout(timeoutId);
|
|
}
|
|
}
|
|
|
|
async function refreshProviderModels(provider: Provider, logger?: FastifyBaseLogger) {
|
|
try {
|
|
const models = await withTimeout(fetchProviderCatalogModels(provider), MODEL_FETCH_TIMEOUT_MS, `${provider} model fetch`);
|
|
modelCatalog[provider] = {
|
|
models,
|
|
loadedAt: new Date().toISOString(),
|
|
error: null,
|
|
};
|
|
logger?.info({ provider, modelCount: models.length }, "model catalog loaded");
|
|
} catch (err: any) {
|
|
const message = err?.message ?? String(err);
|
|
const previous = modelCatalog[provider];
|
|
const fallbackModels = getProviderCatalogFallbackModels(provider);
|
|
modelCatalog[provider] = {
|
|
models: previous?.models.length ? previous.models : fallbackModels,
|
|
loadedAt: previous?.loadedAt ?? null,
|
|
error: message,
|
|
};
|
|
logger?.warn({ provider, err: message }, "failed to load provider model catalog");
|
|
}
|
|
}
|
|
|
|
export async function refreshModelCatalog(logger?: FastifyBaseLogger) {
|
|
if (catalogRefreshPromise) return catalogRefreshPromise;
|
|
|
|
catalogRefreshPromise = Promise.all(listModelCatalogProviders().map((provider) => refreshProviderModels(provider, logger)))
|
|
.then(() => undefined)
|
|
.finally(() => {
|
|
catalogRefreshPromise = null;
|
|
});
|
|
|
|
return catalogRefreshPromise;
|
|
}
|
|
|
|
export async function warmModelCatalog(logger?: FastifyBaseLogger) {
|
|
await refreshModelCatalog(logger);
|
|
}
|
|
|
|
export function startModelCatalogRefreshLoop(logger?: FastifyBaseLogger) {
|
|
const timer = setInterval(() => {
|
|
void refreshModelCatalog(logger);
|
|
}, MODEL_CATALOG_REFRESH_INTERVAL_MS);
|
|
timer.unref?.();
|
|
|
|
return () => {
|
|
clearInterval(timer);
|
|
};
|
|
}
|
|
|
|
export function getModelCatalogSnapshot(): ModelCatalogSnapshot {
|
|
const snapshot: ModelCatalogSnapshot = {};
|
|
for (const provider of listModelCatalogProviders()) {
|
|
const entry = modelCatalog[provider] ?? { models: [], loadedAt: null, error: null };
|
|
snapshot[provider] = {
|
|
models: [...entry.models],
|
|
loadedAt: entry.loadedAt,
|
|
error: entry.error,
|
|
};
|
|
}
|
|
return snapshot;
|
|
}
|