feat: add cross-provider fallback routing with request/response

examples/8-cross-provider-fallback.js

@@ -0,0 +1,88 @@
+/**
+ * Example 8: Cross-Provider Fallback
+ *
+ * Demonstrates automatic fallback across different LLM providers.
+ * If GPT-4o fails, automatically retries with Claude, then Gemini —
+ * all transparent to the caller with a unified budget.
+ *
+ * Prerequisites:
+ *   npm install tokenfirewall
+ *   Set environment variables: OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY
+ */
+
+const {
+  createBudgetGuard,
+  createModelRouter,
+  registerApiKeys,
+  patchGlobalFetch,
+  getBudgetStatus,
+  isCrossProviderEnabled,
+} = require("tokenfirewall");
+
+// 1. Register API keys for all the providers you want to fallback between
+registerApiKeys({
+  openai: process.env.OPENAI_API_KEY,
+  anthropic: process.env.ANTHROPIC_API_KEY,
+  gemini: process.env.GEMINI_API_KEY,
+});
+
+// 2. Create budget guard — costs are tracked across ALL providers
+createBudgetGuard({
+  monthlyLimit: 50, // $50 USD total budget
+  mode: "block",
+});
+
+// 3. Create model router with cross-provider fallback chains
+createModelRouter({
+  strategy: "fallback",
+  fallbackMap: {
+    // If GPT-4o fails → try Claude 3.5 Sonnet → then Gemini 2.5 Pro
+    "gpt-4o": ["claude-3-5-sonnet-20241022", "gemini-2.5-pro"],
+    // If Claude fails → try GPT-4o-mini → then Gemini
+    "claude-3-5-sonnet-20241022": ["gpt-4o-mini", "gemini-2.5-pro"],
+    // If Gemini fails → try GPT-4o-mini → then Claude Haiku
+    "gemini-2.5-pro": ["gpt-4o-mini", "claude-3-5-haiku-20241022"],
+  },
+  maxRetries: 2,
+  enableCrossProvider: true, // <-- enable cross-provider fallback
+});
+
+// 4. Patch global fetch
+patchGlobalFetch();
+
+console.log("Cross-provider enabled:", isCrossProviderEnabled());
+
+// 5. Make a normal API call — fallback is fully transparent
+async function main() {
+  try {
+    const response = await fetch("https://api.openai.com/v1/chat/completions", {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        model: "gpt-4o",
+        messages: [{ role: "user", content: "What is the capital of France?" }],
+        max_tokens: 100,
+      }),
+    });
+
+    const data = await response.json();
+    console.log("\nResponse:", data.choices?.[0]?.message?.content);
+
+    // Budget is tracked across all providers
+    const status = getBudgetStatus();
+    console.log("\nBudget status:", status);
+
+    // If GPT-4o failed, the response was automatically:
+    //   1. Transformed to Claude/Gemini format
+    //   2. Sent to the fallback provider
+    //   3. Response transformed back to OpenAI format
+    //   4. Returned as if GPT-4o answered — fully transparent!
+  } catch (error) {
+    console.error("All providers failed:", error.message);
+  }
+}
+
+main();

src/index.ts

@@ -7,7 +7,8 @@ import { patchProvider } from "./interceptors/sdkInterceptor";
 import { listAvailableModels, ModelInfo, ListModelsOptions } from "./introspection/modelLister";
 import { contextRegistry } from "./introspection/contextRegistry";
 import { ModelRouter } from "./router/modelRouter";
-import { ModelRouterOptions } from "./router/types";
+import { ModelRouterOptions, ApiKeyConfig } from "./router/types";
+import { apiKeyManager } from "./router/apiKeyManager";
 
 let globalBudgetManager: BudgetManager | null = null;
 let globalModelRouter: ModelRouter | null = null;
@@ -135,6 +136,25 @@ export function registerModels(
   }
 }
 
+/**
+ * Register API keys for cross-provider fallback
+ * @param keys - Object mapping provider names to API keys
+ */
+export function registerApiKeys(keys: ApiKeyConfig): void {
+  if (!keys || typeof keys !== 'object') {
+    throw new Error('TokenFirewall: Keys must be an object mapping provider names to API keys');
+  }
+  apiKeyManager.registerKeys(keys);
+}
+
+/**
+ * Check if cross-provider fallback is enabled
+ * @returns true if a model router exists with cross-provider enabled
+ */
+export function isCrossProviderEnabled(): boolean {
+  return globalModelRouter?.isCrossProviderEnabled() ?? false;
+}
+
 /**
  * Get current budget status
  * @returns Budget status or null if no budget guard is active
@@ -228,7 +248,8 @@ export type {
   FailureType,
   FailureContext,
   RoutingDecision,
-  RouterEvent
+  RouterEvent,
+  ApiKeyConfig
 } from "./router/types";
 
 /**

src/interceptors/fetchInterceptor.ts

@@ -3,6 +3,11 @@ import { calculateCost } from "../core/costEngine";
 import { logger } from "../logger";
 import { BudgetManager } from "../core/budgetManager";
 import { ModelRouter } from "../router/modelRouter";
+import { detectProvider, buildProviderUrl, isCrossProviderSwitch } from "../router/providerDetector";
+import { apiKeyManager } from "../router/apiKeyManager";
+import { buildProviderHeaders, appendApiKeyToUrl } from "../router/providerHeaders";
+import { transformRequest } from "../router/requestTransformer";
+import { transformResponse } from "../router/responseTransformer";
 
 let isPatched = false;
 let budgetManager: BudgetManager | null = null;
@@ -98,7 +103,7 @@ async function standardFetch(
 }
 
 /**
- * Fetch with automatic retry and model switching
+ * Fetch with automatic retry and model switching (including cross-provider)
  */
 async function fetchWithRetry(
   input: Parameters<typeof fetch>[0],
@@ -127,12 +132,22 @@ async function fetchWithRetry(
   let currentInput = input;
 
   // Extract original model and provider from request
-  const { originalModel, provider } = extractModelInfo(input, init);
+  const { originalModel, provider: originalProvider } = extractModelInfo(input, init);
 
   if (originalModel) {
     attemptedModels.push(originalModel);
   }
 
+  // Parse the original request body for potential cross-provider transformations
+  let originalRequestBody: any = null;
+  if (init?.body) {
+    try {
+      originalRequestBody = JSON.parse(init.body as string);
+    } catch {
+      // Not JSON
+    }
+  }
+
   while (retryCount <= (modelRouter?.getMaxRetries() || 0)) {
     try {
       // Make the request
@@ -157,12 +172,40 @@ async function fetchWithRetry(
         throw new Error(JSON.stringify(errorObj));
       }
 
+      // If this was a cross-provider retry, transform the response back
+      // to the original provider's format for transparency
+      if (originalProvider && retryCount > 0) {
+        const currentModel = extractCurrentModel(currentInput, currentInit);
+        const currentProvider = currentModel ? detectProvider(currentModel) : null;
+
+        if (currentProvider && currentProvider !== originalProvider) {
+          try {
+            const clonedForTransform = response.clone();
+            const responseData = await clonedForTransform.json();
+            const transformed = transformResponse(
+              responseData,
+              currentProvider,      // source: the provider that actually responded
+              originalProvider,     // target: what the caller expects
+              currentModel || ''
+            );
+            return new Response(JSON.stringify(transformed), {
+              status: response.status,
+              statusText: response.statusText,
+              headers: { 'Content-Type': 'application/json' },
+            });
+          } catch {
+            // If transformation fails, return original response
+            return response;
+          }
+        }
+      }
+
       return response;
     } catch (error) {
       lastError = error;
 
       // If no router or no model info, throw immediately
-      if (!modelRouter || !originalModel || !provider) {
+      if (!modelRouter || !originalModel || !originalProvider) {
         throw error;
       }
 
@@ -187,7 +230,7 @@ async function fetchWithRetry(
             return {};
           }
         })() : {},
-        provider,
+        provider: originalProvider,
         retryCount,
         attemptedModels
       });
@@ -206,11 +249,38 @@ async function fetchWithRetry(
         maxRetries: modelRouter.getMaxRetries()
       });
 
-      // Update request with new model
       attemptedModels.push(decision.nextModel);
-      const updated = updateRequestModel(currentInput, currentInit, decision.nextModel, provider);
-      currentInput = updated.input;
-      currentInit = updated.init;
+
+      // Check if this is a cross-provider switch
+      const nextProvider = detectProvider(decision.nextModel);
+      const isCrossProvider = nextProvider
+        && originalProvider
+        && isCrossProviderSwitch(originalModel, decision.nextModel)
+        && modelRouter.isCrossProviderEnabled();
+
+      if (isCrossProvider && nextProvider && originalRequestBody) {
+        // --- Cross-provider fallback ---
+        const updated = buildCrossProviderRequest(
+          originalRequestBody,
+          originalProvider,
+          nextProvider,
+          decision.nextModel
+        );
+
+        if (updated) {
+          currentInput = updated.url;
+          currentInit = updated.init;
+        } else {
+          // Could not build cross-provider request, throw
+          throw error;
+        }
+      } else {
+        // --- Same-provider fallback (existing behavior) ---
+        const updated = updateRequestModel(currentInput, currentInit, decision.nextModel, originalProvider);
+        currentInput = updated.input;
+        currentInit = updated.init;
+      }
+
       retryCount++;
     }
   }
@@ -221,6 +291,85 @@ async function fetchWithRetry(
   );
 }
 
+/**
+ * Build a completely new request for a different provider (cross-provider fallback)
+ */
+function buildCrossProviderRequest(
+  originalBody: any,
+  sourceProvider: string,
+  targetProvider: string,
+  targetModel: string
+): { url: string; init: RequestInit } | null {
+  // Get API key for target provider
+  const apiKey = apiKeyManager.getKey(targetProvider);
+  if (!apiKey) {
+    console.warn(
+      `TokenFirewall Router: No API key registered for provider "${targetProvider}". ` +
+      `Cross-provider fallback skipped. Register keys with registerApiKeys().`
+    );
+    return null;
+  }
+
+  // Transform request body
+  const transformedBody = transformRequest(
+    originalBody,
+    sourceProvider,
+    targetProvider,
+    targetModel
+  );
+
+  // Build target URL
+  let targetUrl = buildProviderUrl(targetProvider, targetModel);
+  if (!targetUrl) {
+    console.warn(
+      `TokenFirewall Router: Unknown endpoint for provider "${targetProvider}".`
+    );
+    return null;
+  }
+
+  // Append API key to URL if needed (Gemini)
+  targetUrl = appendApiKeyToUrl(targetUrl, targetProvider, apiKey);
+
+  // Build headers
+  const headers = buildProviderHeaders(targetProvider, apiKey);
+
+  return {
+    url: targetUrl,
+    init: {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(transformedBody),
+    },
+  };
+}
+
+/**
+ * Extract current model from the (possibly updated) request
+ */
+function extractCurrentModel(
+  input: Parameters<typeof fetch>[0],
+  init?: Parameters<typeof fetch>[1]
+): string | null {
+  // Check URL for Gemini-style model
+  const url = typeof input === 'string' ? input : (input instanceof Request ? input.url : String(input));
+  const geminiMatch = url.match(/\/models\/([^:?]+)/);
+  if (geminiMatch) {
+    return geminiMatch[1];
+  }
+
+  // Check body for model field
+  if (init?.body) {
+    try {
+      const body = JSON.parse(init.body as string);
+      return body.model || null;
+    } catch {
+      return null;
+    }
+  }
+
+  return null;
+}
+
 /**
  * Extract model and provider information from request
  */
@@ -321,7 +470,6 @@ function updateRequestModel(
         body: init?.body || null,
         mode: input.mode,
         credentials: input.credentials,
-        cache: input.cache,
         redirect: input.redirect,
         referrer: input.referrer,
         integrity: input.integrity