How I Cut My Rork App's AI Costs from $350 to $35/Month with Cloudflare AI Gateway

// workers/ai-proxy/src/index.ts
import { Hono } from 'hono'
import { cors } from 'hono/cors'
 
type Bindings = {
  OPENAI_API_KEY: string
  GEMINI_API_KEY: string
  AI_GATEWAY_ACCOUNT_ID: string
  AI_GATEWAY_ID: string
  RATE_LIMIT: KVNamespace
}
 
type Variables = {
  userId: string
}
 
const app = new Hono<{ Bindings: Bindings; Variables: Variables }>()
 
// CORS — lock this down to your app's actual domain in production
app.use('*', cors({
  origin: (origin) => {
    const allowed = ['https://your-app-domain.com', 'exp://localhost']
    return allowed.includes(origin) ? origin : null
  },
  allowHeaders: ['Content-Type', 'Authorization', 'X-User-ID'],
}))
 
// Auth middleware
app.use('/ai/*', async (c, next) => {
  const userId = c.req.header('X-User-ID')
  if (!userId) {
    return c.json({ error: 'Missing X-User-ID header' }, 401)
  }
  c.set('userId', userId)
  await next()
})
 
// Main text generation endpoint
app.post('/ai/generate', async (c) => {
  const userId = c.get('userId')
  const body = await c.req.json<{
    prompt: string
    model?: string
    cache?: boolean
    maxTokens?: number
  }>()
 
  const {
    prompt,
    model = 'gpt-4o-mini',
    cache = true,
    maxTokens = 1000,
  } = body
 
  // Rate limiting: 50 requests per user per hour
  const hourSlot = Math.floor(Date.now() / 3600000)
  const rateLimitKey = `rl:${userId}:${hourSlot}`
  const currentCount = parseInt(await c.env.RATE_LIMIT.get(rateLimitKey) || '0')
 
  if (currentCount >= 50) {
    const msUntilReset = 3600000 - (Date.now() % 3600000)
    return c.json({
      error: 'Rate limit exceeded',
      retryAfterSeconds: Math.ceil(msUntilReset / 1000),
    }, 429)
  }
 
  // Increment with TTL slightly longer than the hour slot to handle edge cases
  await c.env.RATE_LIMIT.put(rateLimitKey, String(currentCount + 1), {
    expirationTtl: 7200, // 2 hours
  })
 
  const normalizedPrompt = normalizePrompt(prompt)
  const cacheEnabled = cache && isCacheable(normalizedPrompt)
  
  const gatewayBase = `https://gateway.ai.cloudflare.com/v1/${c.env.AI_GATEWAY_ACCOUNT_ID}/${c.env.AI_GATEWAY_ID}`
 
  // Attempt OpenAI first
  const openAIResult = await tryOpenAI({
    prompt: normalizedPrompt,
    model,
    maxTokens,
    cacheEnabled,
    gatewayBase,
    apiKey: c.env.OPENAI_API_KEY,
  })
 
  if (openAIResult.success) {
    return c.json(openAIResult.data)
  }
 
  // Log the failure reason before attempting fallback
  console.warn(`OpenAI failed (${openAIResult.errorCode}), trying Gemini fallback`)
 
  // Only fall back when it makes sense to try a different provider
  if (!shouldFallback(openAIResult.errorCode)) {
    return c.json({ error: openAIResult.errorMessage }, openAIResult.errorCode)
  }
 
  const geminiResult = await tryGemini({
    prompt: normalizedPrompt,
    cacheEnabled,
    gatewayBase,
    apiKey: c.env.GEMINI_API_KEY,
  })
 
  if (geminiResult.success) {
    return c.json(geminiResult.data)
  }
 
  console.error(`All providers failed. OpenAI: ${openAIResult.errorCode}, Gemini: ${geminiResult.errorCode}`)
  return c.json({ error: 'AI service temporarily unavailable' }, 503)
})
 
// Structured OpenAI call with explicit success/failure typing
async function tryOpenAI(params: {
  prompt: string
  model: string
  maxTokens: number
  cacheEnabled: boolean
  gatewayBase: string
  apiKey: string
}): Promise<{ success: true; data: unknown } | { success: false; errorCode: number; errorMessage: string }> {
  try {
    const response = await fetch(`${params.gatewayBase}/openai/chat/completions`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${params.apiKey}`,
        'cf-aig-cache-ttl': params.cacheEnabled ? '86400' : '0',
      },
      body: JSON.stringify({
        model: params.model,
        messages: [{ role: 'user', content: params.prompt }],
        max_tokens: params.maxTokens,
        temperature: params.cacheEnabled ? 0 : 0.7,
      }),
      signal: AbortSignal.timeout(10000),
    })
 
    if (response.ok) {
      return { success: true, data: await response.json() }
    }
 
    let errorMessage = `OpenAI error ${response.status}`
    try {
      const errBody: any = await response.json()
      errorMessage = errBody?.error?.message || errorMessage
    } catch { /* ignore */ }
 
    return { success: false, errorCode: response.status, errorMessage }
 
  } catch (err: any) {
    if (err.name === 'TimeoutError') {
      return { success: false, errorCode: 504, errorMessage: 'OpenAI request timed out' }
    }
    return { success: false, errorCode: 503, errorMessage: 'OpenAI network error' }
  }
}
 
// Gemini call with OpenAI-compatible response format
async function tryGemini(params: {
  prompt: string
  cacheEnabled: boolean
  gatewayBase: string
  apiKey: string
}): Promise<{ success: true; data: unknown } | { success: false; errorCode: number; errorMessage: string }> {
  try {
    const response = await fetch(
      `${params.gatewayBase}/google-ai-studio/v1beta/models/gemini-2.0-flash:generateContent`,
      {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          'x-goog-api-key': params.apiKey,
          'cf-aig-cache-ttl': params.cacheEnabled ? '86400' : '0',
        },
        body: JSON.stringify({
          contents: [{ parts: [{ text: params.prompt }] }],
          generationConfig: { temperature: params.cacheEnabled ? 0 : 0.7 },
        }),
        signal: AbortSignal.timeout(12000),
      }
    )
 
    if (!response.ok) {
      return {
        success: false,
        errorCode: response.status,
        errorMessage: `Gemini error ${response.status}`,
      }
    }
 
    const data: any = await response.json()
    const content = data?.candidates?.[0]?.content?.parts?.[0]?.text
 
    if (!content) {
      return { success: false, errorCode: 500, errorMessage: 'Empty Gemini response' }
    }
 
    // Return in OpenAI-compatible format so the Rork app doesn't need provider-specific handling
    return {
      success: true,
      data: {
        choices: [{ message: { role: 'assistant', content }, finish_reason: 'stop' }],
        _provider: 'gemini',
      },
    }
  } catch (err: any) {
    if (err.name === 'TimeoutError') {
      return { success: false, errorCode: 504, errorMessage: 'Gemini request timed out' }
    }
    return { success: false, errorCode: 503, errorMessage: 'Gemini network error' }
  }
}
 
// Whether a given HTTP error code warrants trying a different provider
function shouldFallback(errorCode: number): boolean {
  // 400: Bad request — the prompt itself is malformed. Gemini won't help.
  if (errorCode === 400) return false
  // 401: Auth error — a key configuration problem, not provider-specific.
  if (errorCode === 401) return false
  // 403: Permissions — same as 401.
  if (errorCode === 403) return false
  // 429: Rate limited — a different provider might have quota remaining.
  if (errorCode === 429) return true
  // 5xx: Server errors — definitely worth trying another provider.
  if (errorCode >= 500) return true
  // 504: Timeout — worth trying.
  if (errorCode === 504) return true
  return false
}
 
// Normalize prompt text to maximize cache hit rate
function normalizePrompt(prompt: string): string {
  return prompt
    .trim()
    .replace(/\s+/g, ' ')
    .replace(/[""]/g, '"')
    .replace(/['']/g, "'")
    .replace(/　/g, ' ')  // Japanese full-width space
    .toLowerCase()
}
 
// Conservative check: only cache when we're confident the output is deterministic enough
function isCacheable(prompt: string): boolean {
  // Personal context signals — don't cache
  const personalSignals = /\b(my|mine|i am|i'm|our|we|yesterday|today|right now|currently|this week)\b/i
  if (personalSignals.test(prompt)) return false
 
  // Too short — context too ambiguous
  if (prompt.length < 40) return false
 
  // Classification, summarization, and extraction are great candidates
  const cacheHints = /(classify|categorize|summarize|translate|extract|tag|label|detect|identify)/i
  if (cacheHints.test(prompt)) return true
 
  // Default: cache it. Conservative analysis above should catch problematic cases.
  return true
}
 
export default app

// hooks/useAI.ts — Drop this into your Rork-generated app
import { useState, useCallback, useRef } from 'react'
import AsyncStorage from '@react-native-async-storage/async-storage'
 
const WORKER_URL = process.env.EXPO_PUBLIC_AI_WORKER_URL!
const LOCAL_CACHE_TTL_MS = 60 * 60 * 1000 // 1 hour
 
interface GenerateOptions {
  prompt: string
  model?: string
  cache?: boolean
}
 
interface GenerateResult {
  content: string
  fromLocalCache: boolean
  provider: string
}
 
export function useAI() {
  const [loading, setLoading] = useState(false)
  const [error, setError] = useState<string | null>(null)
  const controllerRef = useRef<AbortController | null>(null)
 
  const generate = useCallback(async (opts: GenerateOptions): Promise<GenerateResult | null> => {
    const { prompt, model = 'gpt-4o-mini', cache = true } = opts
 
    setLoading(true)
    setError(null)
    controllerRef.current?.abort()
    controllerRef.current = new AbortController()
 
    // Tier 1: AsyncStorage local cache (instant, no network)
    if (cache) {
      try {
        const key = `aicache:${prompt.slice(0, 120)}`
        const raw = await AsyncStorage.getItem(key)
        if (raw) {
          const { data, ts } = JSON.parse(raw)
          if (Date.now() - ts < LOCAL_CACHE_TTL_MS) {
            setLoading(false)
            return { ...data, fromLocalCache: true }
          }
        }
      } catch {
        // Non-fatal, continue to network request
      }
    }
 
    try {
      const userId = await resolveUserId()
 
      const res = await fetch(`${WORKER_URL}/ai/generate`, {
        method: 'POST',
        signal: controllerRef.current.signal,
        headers: {
          'Content-Type': 'application/json',
          'X-User-ID': userId,
        },
        body: JSON.stringify({ prompt, model, cache }),
      })
 
      if (res.status === 429) {
        const { retryAfterSeconds } = await res.json()
        const minutes = Math.ceil(retryAfterSeconds / 60)
        setError(`Request limit reached. Try again in ${minutes} minute${minutes !== 1 ? 's' : ''}.`)
        setLoading(false)
        return null
      }
 
      if (res.status === 503) {
        setError('AI is temporarily unavailable. Please try again shortly.')
        setLoading(false)
        return null
      }
 
      if (!res.ok) {
        throw new Error(`Worker returned ${res.status}`)
      }
 
      const json = await res.json<{ choices: Array<{ message: { content: string } }>; _provider?: string }>()
      const result: GenerateResult = {
        content: json.choices[0].message.content,
        fromLocalCache: false,
        provider: json._provider ?? 'openai',
      }
 
      // Tier 2 write: save to local cache for subsequent requests
      if (cache) {
        const key = `aicache:${prompt.slice(0, 120)}`
        AsyncStorage.setItem(key, JSON.stringify({ data: result, ts: Date.now() })).catch(() => {})
      }
 
      setLoading(false)
      return result
 
    } catch (err: any) {
      if (err.name === 'AbortError') {
        setLoading(false)
        return null
      }
      setError('Something went wrong. Please try again.')
      console.error('[useAI] error:', err)
      setLoading(false)
      return null
    }
  }, [])
 
  const cancel = useCallback(() => controllerRef.current?.abort(), [])
 
  return { generate, loading, error, cancel }
}
 
async function resolveUserId(): Promise<string> {
  const existing = await AsyncStorage.getItem('ai_user_id')
  if (existing) return existing
  const generated = `anon:${Date.now().toString(36)}${Math.random().toString(36).slice(2)}`
  await AsyncStorage.setItem('ai_user_id', generated)
  return generated
}

// Example: AI-powered content categorization in a Rork app
import { useAI } from '../hooks/useAI'
 
export function CategoryTagger({ content }: { content: string }) {
  const { generate, loading, error } = useAI()
  const [category, setCategory] = useState<string | null>(null)
 
  const categorize = async () => {
    const result = await generate({
      prompt: `Categorize the following content into exactly one of these categories: 
               Technology, Health, Finance, Entertainment, Education, Other.
               Respond with only the category name.
               Content: ${content}`,
      cache: true, // Identical content will be served from cache
    })
 
    if (result) {
      setCategory(result.content.trim())
      // Optional: show a subtle indicator if served from local cache
      if (result.fromLocalCache) {
        console.log('Category served from local cache — instant response')
      }
    }
  }
 
  return (
    // ... your UI
  )
}

function getCacheTTL(prompt: string): number {
  // Real-time signals: very short cache
  if (/(trending|popular right now|latest|breaking|current)/i.test(prompt)) {
    return 3600 // 1 hour
  }
  // User-generated content classification: medium cache
  if (/(classify|categorize|tag|label)/i.test(prompt)) {
    return 604800 // 1 week
  }
  // Factual extraction from static content: long cache
  if (/(extract|summarize|translate)/i.test(prompt)) {
    return 2592000 // 30 days
  }
  // Default
  return 86400 // 24 hours
}
 
// Apply in your fetch call:
// 'cf-aig-cache-ttl': String(getCacheTTL(normalizedPrompt))

export async function warmCacheForContent(
  contentText: string,
  gatewayBase: string,
  apiKey: string
): Promise<void> {
  const snippet = contentText.slice(0, 500).toLowerCase().trim().replace(/\s+/g, ' ')
  
  const warmupPrompts = [
    `classify this content into one of: technology, health, finance, entertainment, education, other. content: ${snippet}`,
    `extract the top 3 keywords from this content: ${snippet}`,
    `write a one-sentence summary of this content: ${snippet}`,
  ]
 
  // Fire all warmup requests in parallel — they populate the cache before real users arrive
  await Promise.allSettled(
    warmupPrompts.map(prompt =>
      fetch(`${gatewayBase}/openai/chat/completions`, {
        method: 'POST',
        headers: {
          'Authorization': `Bearer ${apiKey}`,
          'Content-Type': 'application/json',
          'cf-aig-cache-ttl': '86400',
        },
        body: JSON.stringify({
          model: 'gpt-4o-mini',
          messages: [{ role: 'user', content: prompt }],
          temperature: 0,
          max_tokens: 200,
        }),
      })
    )
  )
}

app.post('/ai/generate', async (c) => {
  const userId = c.get('userId')
  const body = await c.req.json()
  const { prompt, cache = true, forceRefresh = false } = body
 
  // Premium users can request fresh results by passing forceRefresh: true
  // This is enforced server-side so free users cannot bypass cache
  const isPremiumUser = userId.startsWith('premium:')
  const bypassCache = forceRefresh && isPremiumUser
  const cacheEnabled = cache && !bypassCache && isCacheable(normalizePrompt(prompt))
 
  // ... rest of implementation
  // In headers: 'cf-aig-cache-ttl': cacheEnabled ? '86400' : '0'
})