Sybil-2/server/src/llm/model-catalog.ts

import type { FastifyBaseLogger } from "fastify";
import {
  fetchProviderCatalogModels,
  getProviderCatalogFallbackModels,
  listModelCatalogProviders,
} from "./provider-adapters.js";
import type { Provider } from "./types.js";

export type ProviderModelSnapshot = {
  models: string[];
  loadedAt: string | null;
  error: string | null;
};

export type ModelCatalogSnapshot = Partial<Record<Provider, ProviderModelSnapshot>>;

const MODEL_FETCH_TIMEOUT_MS = 15000;
const MODEL_CATALOG_REFRESH_INTERVAL_MS = 24 * 60 * 60 * 1000;

const modelCatalog: ModelCatalogSnapshot = {};

let catalogRefreshPromise: Promise<void> | null = null;

async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string) {
  let timeoutId: NodeJS.Timeout | null = null;
  try {
    return await Promise.race([
      promise,
      new Promise<T>((_, reject) => {
        timeoutId = setTimeout(() => {
          reject(new Error(`${label} timed out after ${timeoutMs}ms`));
        }, timeoutMs);
      }),
    ]);
  } finally {
    if (timeoutId) clearTimeout(timeoutId);
  }
}

async function refreshProviderModels(provider: Provider, logger?: FastifyBaseLogger) {
  try {
    const models = await withTimeout(fetchProviderCatalogModels(provider), MODEL_FETCH_TIMEOUT_MS, `${provider} model fetch`);
    modelCatalog[provider] = {
      models,
      loadedAt: new Date().toISOString(),
      error: null,
    };
    logger?.info({ provider, modelCount: models.length }, "model catalog loaded");
  } catch (err: any) {
    const message = err?.message ?? String(err);
    const previous = modelCatalog[provider];
    const fallbackModels = getProviderCatalogFallbackModels(provider);
    modelCatalog[provider] = {
      models: previous?.models.length ? previous.models : fallbackModels,
      loadedAt: previous?.loadedAt ?? null,
      error: message,
    };
    logger?.warn({ provider, err: message }, "failed to load provider model catalog");
  }
}

export async function refreshModelCatalog(logger?: FastifyBaseLogger) {
  if (catalogRefreshPromise) return catalogRefreshPromise;

  catalogRefreshPromise = Promise.all(listModelCatalogProviders().map((provider) => refreshProviderModels(provider, logger)))
    .then(() => undefined)
    .finally(() => {
      catalogRefreshPromise = null;
    });

  return catalogRefreshPromise;
}

export async function warmModelCatalog(logger?: FastifyBaseLogger) {
  await refreshModelCatalog(logger);
}

export function startModelCatalogRefreshLoop(logger?: FastifyBaseLogger) {
  const timer = setInterval(() => {
    void refreshModelCatalog(logger);
  }, MODEL_CATALOG_REFRESH_INTERVAL_MS);
  timer.unref?.();

  return () => {
    clearInterval(timer);
  };
}

export function getModelCatalogSnapshot(): ModelCatalogSnapshot {
  const snapshot: ModelCatalogSnapshot = {};
  for (const provider of listModelCatalogProviders()) {
    const entry = modelCatalog[provider] ?? { models: [], loadedAt: null, error: null };
    snapshot[provider] = {
      models: [...entry.models],
      loadedAt: entry.loadedAt,
      error: entry.error,
    };
  }
  return snapshot;
}