Sybil-2/server/src/llm/model-catalog.ts

import type { FastifyBaseLogger } from "fastify";
import { anthropicClient, openaiClient, xaiClient } from "./providers.js";
import type { Provider } from "./types.js";

export type ProviderModelSnapshot = {
  models: string[];
  loadedAt: string | null;
  error: string | null;
};

export type ModelCatalogSnapshot = Record<Provider, ProviderModelSnapshot>;

const providers: Provider[] = ["openai", "anthropic", "xai"];
const MODEL_FETCH_TIMEOUT_MS = 15000;

const modelCatalog: ModelCatalogSnapshot = {
  openai: { models: [], loadedAt: null, error: null },
  anthropic: { models: [], loadedAt: null, error: null },
  xai: { models: [], loadedAt: null, error: null },
};

function uniqSorted(models: string[]) {
  return [...new Set(models.map((value) => value.trim()).filter(Boolean))].sort((a, b) => a.localeCompare(b));
}

async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string) {
  let timeoutId: NodeJS.Timeout | null = null;
  try {
    return await Promise.race([
      promise,
      new Promise<T>((_, reject) => {
        timeoutId = setTimeout(() => {
          reject(new Error(`${label} timed out after ${timeoutMs}ms`));
        }, timeoutMs);
      }),
    ]);
  } finally {
    if (timeoutId) clearTimeout(timeoutId);
  }
}

async function fetchProviderModels(provider: Provider) {
  if (provider === "openai") {
    const page = await openaiClient().models.list();
    return uniqSorted(page.data.map((model) => model.id));
  }

  if (provider === "anthropic") {
    const page = await anthropicClient().models.list({ limit: 200 });
    return uniqSorted(page.data.map((model) => model.id));
  }

  const page = await xaiClient().models.list();
  return uniqSorted(page.data.map((model) => model.id));
}

async function refreshProviderModels(provider: Provider, logger?: FastifyBaseLogger) {
  try {
    const models = await withTimeout(fetchProviderModels(provider), MODEL_FETCH_TIMEOUT_MS, `${provider} model fetch`);
    modelCatalog[provider] = {
      models,
      loadedAt: new Date().toISOString(),
      error: null,
    };
    logger?.info({ provider, modelCount: models.length }, "model catalog loaded");
  } catch (err: any) {
    const message = err?.message ?? String(err);
    modelCatalog[provider] = {
      models: [],
      loadedAt: new Date().toISOString(),
      error: message,
    };
    logger?.warn({ provider, err: message }, "failed to load provider model catalog");
  }
}

export async function warmModelCatalog(logger?: FastifyBaseLogger) {
  await Promise.all(providers.map((provider) => refreshProviderModels(provider, logger)));
}

export function getModelCatalogSnapshot(): ModelCatalogSnapshot {
  return {
    openai: {
      models: [...modelCatalog.openai.models],
      loadedAt: modelCatalog.openai.loadedAt,
      error: modelCatalog.openai.error,
    },
    anthropic: {
      models: [...modelCatalog.anthropic.models],
      loadedAt: modelCatalog.anthropic.loadedAt,
      error: modelCatalog.anthropic.error,
    },
    xai: {
      models: [...modelCatalog.xai.models],
      loadedAt: modelCatalog.xai.loadedAt,
      error: modelCatalog.xai.error,
    },
  };
}