to Production Edge AI in Rork Apps— Ollama Streaming, Conversation History, and Cost Architecture

// hooks/useOllamaStream.js
import { useState, useCallback, useRef } from 'react';
 
const OLLAMA_HOST = __DEV__
  ? 'http://192.168.1.100:11434'        // dev: direct local IP
  : 'https://your-ollama-gateway.com';  // prod: VPS gateway
 
export function useOllamaStream() {
  const [streamingText, setStreamingText] = useState('');
  const [isStreaming, setIsStreaming] = useState(false);
  const abortControllerRef = useRef(null);
 
  const streamChat = useCallback(async (messages, onComplete) => {
    abortControllerRef.current = new AbortController();
    setIsStreaming(true);
    setStreamingText('');
 
    let fullText = '';
 
    try {
      const response = await fetch(`${OLLAMA_HOST}/api/chat`, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
          model: 'gemma4:7b',
          messages,
          stream: true,
          options: { temperature: 0.7, top_p: 0.9, num_predict: 1024 },
        }),
        signal: abortControllerRef.current.signal,
      });
 
      if (!response.ok) throw new Error(`HTTP ${response.status}`);
 
      // Parse NDJSON (newline-delimited JSON) stream
      const reader = response.body.getReader();
      const decoder = new TextDecoder();
      let buffer = '';
 
      while (true) {
        const { done, value } = await reader.read();
        if (done) break;
 
        buffer += decoder.decode(value, { stream: true });
        const lines = buffer.split('\n');
        buffer = lines.pop() ?? '';
 
        for (const line of lines) {
          if (!line.trim()) continue;
          try {
            const parsed = JSON.parse(line);
            if (parsed.message?.content) {
              fullText += parsed.message.content;
              setStreamingText(fullText);
            }
            if (parsed.done) { onComplete?.(fullText); return; }
          } catch { /* ignore malformed JSON chunks */ }
        }
      }
    } catch (error) {
      if (error.name === 'AbortError') {
        onComplete?.(fullText); // user cancel — treat as normal completion
      } else {
        throw error;
      }
    } finally {
      setIsStreaming(false);
    }
  }, []);
 
  const cancelStream = useCallback(() => {
    abortControllerRef.current?.abort();
  }, []);
 
  return { streamingText, isStreaming, streamChat, cancelStream };
}

// services/ConversationDB.js
import * as SQLite from 'expo-sqlite';
 
class ConversationDB {
  async init() {
    this.db = await SQLite.openDatabaseAsync('conversations.db');
    await this.db.execAsync(`
      CREATE TABLE IF NOT EXISTS conversations (
        id TEXT PRIMARY KEY,
        title TEXT,
        model TEXT,
        created_at INTEGER,
        updated_at INTEGER
      );
      CREATE TABLE IF NOT EXISTS messages (
        id TEXT PRIMARY KEY,
        conversation_id TEXT,
        role TEXT CHECK(role IN ('user', 'assistant', 'system')),
        content TEXT,
        token_count INTEGER DEFAULT 0,
        created_at INTEGER,
        FOREIGN KEY (conversation_id) REFERENCES conversations(id) ON DELETE CASCADE
      );
      CREATE INDEX IF NOT EXISTS idx_messages_conv
        ON messages(conversation_id, created_at);
    `);
  }
 
  async createConversation(model = 'gemma4:7b') {
    const id = `conv_${Date.now()}_${Math.random().toString(36).slice(2)}`;
    const now = Date.now();
    await this.db.runAsync(
      'INSERT INTO conversations VALUES (?, ?, ?, ?, ?)',
      [id, 'New conversation', model, now, now]
    );
    return id;
  }
 
  async addMessage(conversationId, role, content, tokenCount = 0) {
    const id = `msg_${Date.now()}_${Math.random().toString(36).slice(2)}`;
    await this.db.runAsync(
      'INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?)',
      [id, conversationId, role, content, tokenCount, Date.now()]
    );
 
    // Auto-generate conversation title from first user message
    if (role === 'user') {
      const { cnt } = await this.db.getFirstAsync(
        'SELECT COUNT(*) as cnt FROM messages WHERE conversation_id = ? AND role = "user"',
        [conversationId]
      );
      if (cnt === 1) {
        const title = content.length > 40 ? content.slice(0, 40) + '…' : content;
        await this.db.runAsync(
          'UPDATE conversations SET title = ?, updated_at = ? WHERE id = ?',
          [title, Date.now(), conversationId]
        );
      }
    }
  }
 
  /**
   * Returns messages that fit within the context window.
   * Trims oldest messages first when token budget is exceeded.
   */
  async getMessagesForContext(conversationId, maxTokens = 3000) {
    const messages = await this.db.getAllAsync(
      `SELECT role, content, token_count FROM messages
       WHERE conversation_id = ?
       ORDER BY created_at DESC LIMIT 50`,
      [conversationId]
    );
 
    let totalTokens = 0;
    const context = [];
 
    for (const msg of messages) {
      // Rough estimate: ~3.5 chars per token for English
      const estimated = msg.token_count || Math.ceil(msg.content.length / 3.5);
      if (totalTokens + estimated > maxTokens) break;
      totalTokens += estimated;
      context.unshift({ role: msg.role, content: msg.content });
    }
 
    return context;
  }
}
 
export const conversationDB = new ConversationDB();

// services/AIRouter.js
const LOCAL_TIMEOUT_MS = 3000;
 
async function checkLocalAvailability(host) {
  try {
    const controller = new AbortController();
    setTimeout(() => controller.abort(), LOCAL_TIMEOUT_MS);
    const res = await fetch(`${host}/api/version`, {
      signal: controller.signal,
    });
    return res.ok;
  } catch { return false; }
}
 
export class AIRouter {
  constructor({ localHost, cloudApiKey, onRouteChange }) {
    this.localHost = localHost;
    this.cloudApiKey = cloudApiKey;
    this.onRouteChange = onRouteChange;
    this.currentRoute = 'unknown';
    this.lastChecked = 0;
    this.CHECK_INTERVAL_MS = 30_000;
  }
 
  async resolveRoute() {
    const now = Date.now();
    // Cache the check result for 30 seconds to avoid hammering the server
    if (now - this.lastChecked < this.CHECK_INTERVAL_MS) {
      return this.currentRoute;
    }
    this.lastChecked = now;
    const available = await checkLocalAvailability(this.localHost);
    const route = available ? 'local' : 'cloud';
    if (route !== this.currentRoute) {
      this.currentRoute = route;
      this.onRouteChange?.(route);
    }
    return route;
  }
 
  async chatCloud(messages) {
    const response = await fetch('https://api.openai.com/v1/chat/completions', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        Authorization: `Bearer ${this.cloudApiKey}`,
      },
      body: JSON.stringify({
        model: 'gpt-4o-mini', // cheapest capable model for fallback
        messages,
        max_tokens: 1024,
      }),
    });
    const data = await response.json();
    return { text: data.choices[0].message.content, source: 'cloud' };
  }
}

# gateway/main.py
from fastapi import FastAPI, HTTPException, Header
from fastapi.responses import StreamingResponse
from pydantic import BaseModel
import httpx
 
app = FastAPI()
OLLAMA_BASE = "http://localhost:11434"
 
class ChatRequest(BaseModel):
    messages: list[dict]
    model: str = "gemma4:7b"
    stream: bool = True
    max_tokens: int = 1024
 
@app.post("/v1/chat")
async def chat(req: ChatRequest, x_api_key: str = Header(None)):
    if x_api_key != "your-secret-gateway-key":
        raise HTTPException(status_code=401)
 
    payload = {
        "model": req.model,
        "messages": req.messages,
        "stream": req.stream,
        "options": {"num_predict": req.max_tokens, "temperature": 0.7},
    }
 
    if req.stream:
        async def generate():
            async with httpx.AsyncClient(timeout=120.0) as client:
                async with client.stream("POST", f"{OLLAMA_BASE}/api/chat", json=payload) as r:
                    async for line in r.aiter_lines():
                        if line:
                            yield line + "\n"
        return StreamingResponse(generate(), media_type="application/x-ndjson")
    else:
        async with httpx.AsyncClient(timeout=60.0) as client:
            r = await client.post(f"{OLLAMA_BASE}/api/chat", json=payload)
            return r.json()
 
@app.get("/health")
async def health():
    try:
        async with httpx.AsyncClient(timeout=3.0) as client:
            r = await client.get(f"{OLLAMA_BASE}/api/version")
            return {"status": "ok", "ollama": r.json()}
    except Exception as e:
        return {"status": "degraded", "error": str(e)}

// config/tiers.js
export const TIERS = {
  free: {
    name: 'Free',
    dailyMessages: 20,
    model: 'gemma4:4b',        // smaller model for free tier
    maxContextMessages: 10,
    cloudFallback: false,      // no fallback = no variable cost
  },
  premium: {
    name: 'Premium',
    dailyMessages: Infinity,   // unlimited — sustainable because cost is fixed
    model: 'gemma4:7b',
    maxContextMessages: 50,
    cloudFallback: true,
    price: { usd: 5, jpy: 580 },
  },
};

async function summarizeOldMessages(convId) {
  const all = await conversationDB.getAllMessages(convId);
  if (all.length < 30) return;
 
  const toSummarize = all.slice(0, -20);
  const summary = await router.chat([
    { role: 'system', content: 'Summarize the following conversation in 3 sentences, preserving key facts and decisions.' },
    { role: 'user', content: JSON.stringify(toSummarize) },
  ]);
  await conversationDB.setSystemPrompt(convId, `[Conversation summary]\n${summary.text}`);
}