LLM Cost Optimization: Cut Your OpenAI & Claude API Spending by 60%

// Model routing by task type
const MODEL_ROUTER = {
  // Simple tasks — GPT-4o-mini is 16x cheaper, near-identical quality
  classify_intent: 'gpt-4o-mini',
  extract_entities: 'gpt-4o-mini',
  summarize_document: 'gpt-4o-mini',
  sentiment_analysis: 'gpt-4o-mini',
  faq_matching: 'gpt-4o-mini',

  // Complex tasks — need flagship model capability
  complex_reasoning: 'gpt-4o',
  code_generation: 'gpt-4o',
  multi_step_analysis: 'gpt-4o',
  creative_writing: 'gpt-4o',
} as const;

async function callLLM(task: keyof typeof MODEL_ROUTER, prompt: string) {
  const model = MODEL_ROUTER[task];
  return openai.chat.completions.create({
    model,
    messages: [{ role: 'user', content: prompt }],
    max_tokens: task.startsWith('extract') ? 200 : 1000, // tight limits for simple tasks
  });
}

// Check if caching is working — look for cached_tokens in usage
const response = await openai.chat.completions.create({
  model: 'gpt-4o',
  messages: [
    { role: 'system', content: LARGE_SYSTEM_PROMPT }, // 2000+ tokens — will be cached
    { role: 'user', content: userMessage },
  ],
});

// Check cache hit rate
const { prompt_tokens, completion_tokens } = response.usage;
const cached = response.usage.prompt_tokens_details?.cached_tokens ?? 0;
const cacheRate = cached / prompt_tokens;
console.log(`Cache hit rate: ${(cacheRate * 100).toFixed(1)}%`);
// Target: >60% cache hit rate for production system prompts

import Anthropic from '@anthropic-ai/sdk';
const anthropic = new Anthropic();

const response = await anthropic.messages.create({
  model: 'claude-sonnet-4-5',
  max_tokens: 1024,
  system: [
    {
      type: 'text',
      text: LARGE_SYSTEM_PROMPT, // 2000+ tokens
      cache_control: { type: 'ephemeral' }, // Mark for caching
    },
  ],
  messages: [
    {
      role: 'user',
      content: [
        {
          type: 'text',
          text: LARGE_DOCUMENT, // Document to analyze — also cache it
          cache_control: { type: 'ephemeral' },
        },
        { type: 'text', text: userQuestion },
      ],
    },
  ],
});

// Check cache savings
const { cache_read_input_tokens, cache_creation_input_tokens } = response.usage;
console.log('Cache reads (cheap):', cache_read_input_tokens);
console.log('Cache writes (one-time):', cache_creation_input_tokens);

// Create a batch job (JSONL input)
import fs from 'fs';

// Build batch requests file
const requests = documents.map((doc, i) => ({
  custom_id: `doc-${i}`,
  method: 'POST',
  url: '/v1/chat/completions',
  body: {
    model: 'gpt-4o-mini',
    messages: [
      { role: 'system', content: 'Classify this document. Respond with JSON.' },
      { role: 'user', content: doc.text },
    ],
    max_tokens: 100,
  },
}));

fs.writeFileSync('batch_input.jsonl', requests.map(r => JSON.stringify(r)).join('\n'));

// Upload and create batch
const file = await openai.files.create({
  file: fs.createReadStream('batch_input.jsonl'),
  purpose: 'batch',
});

const batch = await openai.batches.create({
  input_file_id: file.id,
  endpoint: '/v1/chat/completions',
  completion_window: '24h',
});

console.log('Batch ID:', batch.id);
// Poll /v1/batches/{batch_id} for completion

// Track cost per request in your app
function calculateCost(usage: {
  prompt_tokens: number;
  completion_tokens: number;
  cached_tokens?: number;
}, model: string): number {
  const PRICING: Record<string, { input: number; output: number; cached?: number }> = {
    'gpt-4o': { input: 0.0000025, output: 0.00001, cached: 0.00000125 },
    'gpt-4o-mini': { input: 0.00000015, output: 0.0000006, cached: 0.000000075 },
    'claude-sonnet-4-5': { input: 0.000003, output: 0.000015, cached: 0.0000003 },
  };

  const pricing = PRICING[model];
  if (!pricing) return 0;

  const cachedTokens = usage.cached_tokens ?? 0;
  const regularInput = usage.prompt_tokens - cachedTokens;

  return (
    regularInput * pricing.input +
    (cachedTokens * (pricing.cached ?? pricing.input)) +
    usage.completion_tokens * pricing.output
  );
}

// Log to your analytics/metrics system
const cost = calculateCost(response.usage, model);
metrics.increment('llm.cost.usd', cost, { model, feature: 'chat' });
metrics.increment('llm.tokens.input', response.usage.prompt_tokens, { model });
metrics.increment('llm.tokens.output', response.usage.completion_tokens, { model });

// Simple multi-provider fallback with cost awareness
async function generateWithFallback(prompt: string, feature: string) {
  const providers = [
    { client: openai, model: 'gpt-4o-mini', costMultiplier: 1 },      // Primary (cheapest)
    { client: anthropic, model: 'claude-haiku-4-5', costMultiplier: 5 }, // Secondary
    { client: openai, model: 'gpt-4o', costMultiplier: 16 },            // Last resort
  ];

  for (const provider of providers) {
    try {
      const result = await callProvider(provider, prompt);
      metrics.increment('llm.provider.success', 1, {
        model: provider.model,
        feature
      });
      return result;
    } catch (err) {
      if (err.status === 429 || err.status >= 500) {
        continue; // Try next provider
      }
      throw err; // Non-retriable error
    }
  }
  throw new Error('All LLM providers failed');
}