Sybil-2/server/src/llm/model-catalog.ts

import type { FastifyBaseLogger } from "fastify";
import { env } from "../env.js";
import { anthropicClient, hermesAgentClient, isHermesAgentConfigured, openaiClient, xaiClient } from "./providers.js";
import type { Provider } from "./types.js";

export type ProviderModelSnapshot = {
  models: string[];
  loadedAt: string | null;
  error: string | null;
};

export type ModelCatalogSnapshot = Partial<Record<Provider, ProviderModelSnapshot>>;

const baseProviders: Provider[] = ["openai", "anthropic", "xai"];
const MODEL_FETCH_TIMEOUT_MS = 15000;
const MODEL_CATALOG_REFRESH_INTERVAL_MS = 24 * 60 * 60 * 1000;

const modelCatalog: ModelCatalogSnapshot = {
  openai: { models: [], loadedAt: null, error: null },
  anthropic: { models: [], loadedAt: null, error: null },
  xai: { models: [], loadedAt: null, error: null },
};

let catalogRefreshPromise: Promise<void> | null = null;

function getCatalogProviders(): Provider[] {
  return isHermesAgentConfigured() ? [...baseProviders, "hermes-agent"] : baseProviders;
}

function uniqSorted(models: string[]) {
  return [...new Set(models.map((value) => value.trim()).filter(Boolean))].sort((a, b) => a.localeCompare(b));
}

function isLikelyOpenAIResponsesModel(model: string) {
  const id = model.toLowerCase();
  if (id.includes("embedding") || id.includes("moderation")) return false;
  if (id.includes("audio") || id.includes("realtime") || id.includes("transcribe") || id.includes("tts")) return false;
  if (id.includes("image") || id.includes("dall-e") || id.includes("sora")) return false;
  if (id.includes("search") || id.includes("computer-use")) return false;
  return /^(gpt-|o\d|chatgpt-)/.test(id);
}

async function withTimeout<T>(promise: Promise<T>, timeoutMs: number, label: string) {
  let timeoutId: NodeJS.Timeout | null = null;
  try {
    return await Promise.race([
      promise,
      new Promise<T>((_, reject) => {
        timeoutId = setTimeout(() => {
          reject(new Error(`${label} timed out after ${timeoutMs}ms`));
        }, timeoutMs);
      }),
    ]);
  } finally {
    if (timeoutId) clearTimeout(timeoutId);
  }
}

async function fetchProviderModels(provider: Provider) {
  if (provider === "openai") {
    const page = await openaiClient().models.list();
    return uniqSorted(page.data.map((model) => model.id).filter(isLikelyOpenAIResponsesModel));
  }

  if (provider === "anthropic") {
    const page = await anthropicClient().models.list({ limit: 200 });
    return uniqSorted(page.data.map((model) => model.id));
  }

  if (provider === "xai") {
    const page = await xaiClient().models.list();
    return uniqSorted(page.data.map((model) => model.id));
  }

  const page = await hermesAgentClient().models.list();
  const models = page.data.map((model) => model.id);
  if (env.HERMES_AGENT_MODEL) models.push(env.HERMES_AGENT_MODEL);
  return uniqSorted(models);
}

async function refreshProviderModels(provider: Provider, logger?: FastifyBaseLogger) {
  try {
    const models = await withTimeout(fetchProviderModels(provider), MODEL_FETCH_TIMEOUT_MS, `${provider} model fetch`);
    modelCatalog[provider] = {
      models,
      loadedAt: new Date().toISOString(),
      error: null,
    };
    logger?.info({ provider, modelCount: models.length }, "model catalog loaded");
  } catch (err: any) {
    const message = err?.message ?? String(err);
    const previous = modelCatalog[provider];
    const fallbackModels = provider === "hermes-agent" && env.HERMES_AGENT_MODEL ? [env.HERMES_AGENT_MODEL] : [];
    modelCatalog[provider] = {
      models: previous?.models.length ? previous.models : fallbackModels,
      loadedAt: previous?.loadedAt ?? null,
      error: message,
    };
    logger?.warn({ provider, err: message }, "failed to load provider model catalog");
  }
}

export async function refreshModelCatalog(logger?: FastifyBaseLogger) {
  if (catalogRefreshPromise) return catalogRefreshPromise;

  catalogRefreshPromise = Promise.all(getCatalogProviders().map((provider) => refreshProviderModels(provider, logger)))
    .then(() => undefined)
    .finally(() => {
      catalogRefreshPromise = null;
    });

  return catalogRefreshPromise;
}

export async function warmModelCatalog(logger?: FastifyBaseLogger) {
  await refreshModelCatalog(logger);
}

export function startModelCatalogRefreshLoop(logger?: FastifyBaseLogger) {
  const timer = setInterval(() => {
    void refreshModelCatalog(logger);
  }, MODEL_CATALOG_REFRESH_INTERVAL_MS);
  timer.unref?.();

  return () => {
    clearInterval(timer);
  };
}

export function getModelCatalogSnapshot(): ModelCatalogSnapshot {
  const snapshot: ModelCatalogSnapshot = {};
  for (const provider of getCatalogProviders()) {
    const entry = modelCatalog[provider] ?? { models: [], loadedAt: null, error: null };
    snapshot[provider] = {
      models: [...entry.models],
      loadedAt: entry.loadedAt,
      error: entry.error,
    };
  }
  return snapshot;
}