Fallback models and circuit breakers

src/pipeline.js

// The whole generator, rebuilt on LangChain chat models:
//
//   (optional) research → outline → write the deck → compile → repair what's broken
//
// It replaces generate.js. The steps are the same; what changed is that each
// step receives a `model`, so we can swap in a different model, or a fake one in tests.
//
// Every model call goes through `call()`: a timeout per attempt, retries with
// backoff for errors worth retrying, and a fallback model when one is unhealthy.

import { HumanMessage, SystemMessage } from "@langchain/core/messages";
import { contentText, createModel } from "./llm.js";
import { generateOutline } from "./outline.js";
import { researchTopic } from "./research.js";
import { buildSystemPrompt, extractTag } from "./prompt.js";
import { LIBRARY } from "./library.js";
import { compileDeck } from "./compile.js";
import { withRetry, withTimeout } from "./retry.js";
import { createModelPool } from "./fallback.js";

const MAX_REPAIR_ROUNDS = 2;
const CALL_TIMEOUT_MS = 90_000; // one model call
const DECK_DEADLINE_MS = 240_000; // the whole deck, inside a typical 300s serverless limit

/** The models to use, best first. The second is lighter and usually less busy. */
export function defaultModels() {
  return [createModel(), createModel({ model: "gemini-flash-lite-latest" })];
}

/** Outline → SlideML, following the component library. */
export async function writeDeck(model, outline, { signal } = {}) {
  const reply = await model.invoke(
    [new SystemMessage(buildSystemPrompt(LIBRARY)), new HumanMessage(`Write the deck for this outline:\n${JSON.stringify(outline, null, 2)}`)],
    { signal },
  );
  const xml = extractTag(contentText(reply), "deck");
  if (!xml) throw new Error("The model didn't reply with a <deck> element.");
  return xml;
}

/** Send the deck back with the compiler's problems and ask for a corrected deck. */
export async function repairDeck(model, xml, problems, { signal } = {}) {
  const list = problems.map((problem) => `- Slide ${problem.slide}: ${problem.message}`).join("\n");
  const reply = await model.invoke(
    [
      new SystemMessage(buildSystemPrompt(LIBRARY)),
      new HumanMessage(`This deck has problems:\n${list}\n\nFix them and reply with the whole corrected <deck>.\n\n${xml}`),
    ],
    { signal },
  );
  return extractTag(contentText(reply), "deck");
}

/**
 * options:
 *   models      chat models, best first (default: defaultModels()); or `model` for just one
 *   research    true to search the web first
 *   slideCount  how many slides to plan (default 6)
 *   theme       "light" or "dark"
 *   compile     the compiler to use (tests pass a fake one)
 *   onEvent     called with { type: "retry" | "fallback", … } so a UI can show what's happening
 *   wait        how retries wait (tests pass a fake one)
 */
export async function generateDeck(topic, options = {}) {
  const { research = false, slideCount = 6, theme, compile = compileDeck, onEvent = () => {}, wait } = options;
  const models = options.model ? [options.model] : (options.models ?? defaultModels());

  const pool = createModelPool(models, { onFallback: (info) => onEvent({ type: "fallback", ...info }) });
  const deadline = Date.now() + DECK_DEADLINE_MS;

  // fallback ⟶ retries ⟶ timeout ⟶ the actual call
  const call = (stage, fn) =>
    pool.run((model) =>
      withRetry(() => withTimeout((signal) => fn(model, signal), CALL_TIMEOUT_MS), {
        retries: 2,
        deadline,
        wait,
        onRetry: (info) => onEvent({ type: "retry", stage, ...info }),
      }),
    );

  const findings = research ? await call("research", (model, signal) => researchTopic(model, topic, { signal })) : { notes: "", sources: [] };
  const outline = await call("outline", (model, signal) => generateOutline(model, topic, { slideCount, research: findings.notes, signal }));
  let xml = await call("deck", (model, signal) => writeDeck(model, outline, { signal }));
  let result = compile(xml, theme);
  let repairs = 0;

  while (result.problems.length > 0 && repairs < MAX_REPAIR_ROUNDS) {
    repairs += 1;
    const fixed = await call("repair", (model, signal) => repairDeck(model, xml, result.problems, { signal }));
    if (!fixed) break;
    const next = compile(fixed, theme);
    if (next.problems.length >= result.problems.length) break;
    xml = fixed;
    result = next;
  }

  return { outline, xml, slides: result.slides, problems: result.problems, repairs, sources: findings.sources };
}