Building an AI Reading Assistant App with Rork — Bookshelf Scanning, Progress Tracking, and Chapter Summaries

// app/(tabs)/scan.tsx
import { useState } from 'react';
import { View, Text, Pressable, Image, ActivityIndicator } from 'react-native';
import * as ImagePicker from 'expo-image-picker';
import * as ImageManipulator from 'expo-image-manipulator';
import { scanBookshelfApi } from '@/lib/api/bookshelf';
 
type DetectedBook = {
  title: string;
  author?: string;
  confidence: number;
};
 
export default function BookshelfScanScreen() {
  const [image, setImage] = useState<string | null>(null);
  const [books, setBooks] = useState<DetectedBook[] | null>(null);
  const [loading, setLoading] = useState(false);
  const [error, setError] = useState<string | null>(null);
 
  const pickAndScan = async () => {
    try {
      setError(null);
 
      const result = await ImagePicker.launchCameraAsync({
        mediaTypes: ImagePicker.MediaTypeOptions.Images,
        quality: 0.9,
        allowsEditing: false,
      });
 
      if (result.canceled) return;
 
      // Resize to 2048px wide to balance OCR quality and upload cost
      const manipulated = await ImageManipulator.manipulateAsync(
        result.assets[0].uri,
        [{ resize: { width: 2048 } }],
        { compress: 0.85, format: ImageManipulator.SaveFormat.JPEG, base64: true },
      );
 
      setImage(manipulated.uri);
      setLoading(true);
 
      const detected = await scanBookshelfApi(manipulated.base64\!);
      setBooks(detected);
    } catch (e: any) {
      // Show a short, friendly message to the user; send the details to Sentry
      setError('We could not read the shelf. Try a different angle without glare.');
      console.error('[BookshelfScan]', e);
    } finally {
      setLoading(false);
    }
  };
 
  return (
    <View className="flex-1 items-center justify-center p-4">
      <Pressable
        onPress={pickAndScan}
        className="bg-indigo-600 px-6 py-4 rounded-full"
        accessibilityLabel="Scan a bookshelf to register books"
      >
        <Text className="text-white font-semibold">Scan bookshelf</Text>
      </Pressable>
 
      {image && <Image source={{ uri: image }} className="w-full h-64 mt-4 rounded-lg" />}
      {loading && <ActivityIndicator className="mt-4" />}
      {error && <Text className="text-red-500 mt-4">{error}</Text>}
      {books && (
        <View className="w-full mt-4">
          <Text className="font-semibold mb-2">Recognized {books.length} books</Text>
          {books.map((b, i) => (
            <Text key={i} className="text-sm text-gray-700">
              {b.title}
              {b.author ? ` · ${b.author}` : ''} (confidence {Math.round(b.confidence * 100)}%)
            </Text>
          ))}
        </View>
      )}
    </View>
  );
}

// workers/src/routes/bookshelf.ts
import { Hono } from 'hono';
import Anthropic from '@anthropic-ai/sdk';
import { GoogleGenerativeAI } from '@google/generative-ai';
 
type Env = {
  ANTHROPIC_API_KEY: string;
  GEMINI_API_KEY: string;
};
 
type DetectedBook = { title: string; author?: string; confidence: number };
 
const app = new Hono<{ Bindings: Env }>();
 
app.post('/api/bookshelf/scan', async (c) => {
  const { imageBase64 } = await c.req.json<{ imageBase64: string }>();
 
  if (\!imageBase64 || imageBase64.length < 1000) {
    return c.json({ error: 'Invalid image data' }, 400);
  }
 
  try {
    const [claudeResult, geminiResult] = await Promise.allSettled([
      detectBooksWithClaude(imageBase64, c.env.ANTHROPIC_API_KEY),
      detectBooksWithGemini(imageBase64, c.env.GEMINI_API_KEY),
    ]);
 
    const merged = mergeDetections(
      claudeResult.status === 'fulfilled' ? claudeResult.value : [],
      geminiResult.status === 'fulfilled' ? geminiResult.value : [],
    );
 
    return c.json({ books: merged });
  } catch (e) {
    console.error('[BookshelfScan] unexpected error', e);
    return c.json({ error: 'Scan failed' }, 500);
  }
});
 
async function detectBooksWithClaude(base64: string, apiKey: string): Promise<DetectedBook[]> {
  const client = new Anthropic({ apiKey });
 
  const response = await client.messages.create({
    model: 'claude-sonnet-4-6',
    max_tokens: 2000,
    messages: [
      {
        role: 'user',
        content: [
          {
            type: 'image',
            source: { type: 'base64', media_type: 'image/jpeg', data: base64 },
          },
          {
            type: 'text',
            text: `Identify titles and authors visible on the spines in this bookshelf image.
Return ONLY a JSON array in this exact shape:
[{"title": "...", "author": "name or null", "confidence": 0.0-1.0}]
 
If a spine is unreadable, omit that book. Never invent titles — confidence must reflect what you actually see.`,
          },
        ],
      },
    ],
  });
 
  const text = response.content[0].type === 'text' ? response.content[0].text : '';
  return parseJsonArray<DetectedBook>(text);
}
 
async function detectBooksWithGemini(base64: string, apiKey: string): Promise<DetectedBook[]> {
  const genAI = new GoogleGenerativeAI(apiKey);
  const model = genAI.getGenerativeModel({ model: 'gemini-2.5-flash' });
 
  const result = await model.generateContent([
    { inlineData: { data: base64, mimeType: 'image/jpeg' } },
    `Return a JSON array of titles and authors visible on the spines: [{"title": "...", "author": "...", "confidence": 0.0-1.0}]`,
  ]);
 
  return parseJsonArray<DetectedBook>(result.response.text());
}
 
function parseJsonArray<T>(raw: string): T[] {
  // Models sometimes wrap output with ```json ... ``` or add preamble; slice the first [...] block
  const match = raw.match(/\[[\s\S]*\]/);
  if (\!match) return [];
  try {
    return JSON.parse(match[0]);
  } catch {
    return [];
  }
}
 
function mergeDetections(a: DetectedBook[], b: DetectedBook[]): DetectedBook[] {
  const merged = [...a];
  for (const book of b) {
    const similar = merged.find(
      (m) => normalizeTitle(m.title) === normalizeTitle(book.title),
    );
    if (\!similar) {
      merged.push(book);
    } else if (book.confidence > similar.confidence) {
      similar.confidence = Math.min(1, (similar.confidence + book.confidence) / 2 + 0.1);
      similar.author = similar.author ?? book.author;
    }
  }
  return merged
    .filter((b) => b.confidence >= 0.5)
    .sort((x, y) => y.confidence - x.confidence);
}
 
function normalizeTitle(t: string): string {
  return t.toLowerCase().replace(/[\s・：:|\-—–]/g, '');
}
 
export default app;

// workers/src/routes/chapters.ts
import { Hono } from 'hono';
import Anthropic from '@anthropic-ai/sdk';
 
type Chapter = { index: number; title: string; estimatedPages?: [number, number] };
 
const app = new Hono<{ Bindings: { ANTHROPIC_API_KEY: string } }>();
 
app.post('/api/books/:bookId/chapters', async (c) => {
  const bookId = c.req.param('bookId');
  const { title, author, totalPages } = await c.req.json<{
    title: string;
    author?: string;
    totalPages?: number;
  }>();
 
  if (\!title || title.length > 200) {
    return c.json({ error: 'Invalid title' }, 400);
  }
 
  const client = new Anthropic({ apiKey: c.env.ANTHROPIC_API_KEY });
 
  const response = await client.messages.create({
    model: 'claude-sonnet-4-6',
    max_tokens: 1500,
    messages: [
      {
        role: 'user',
        content: `Estimate the chapter structure of the book "${title}"${author ? ` by ${author}` : ''}.
${totalPages ? `The total page count is approximately ${totalPages}. Include estimated page ranges for each chapter.` : ''}
Return ONLY a JSON array in this shape:
[{"index": 1, "title": "Chapter title", "estimatedPages": [1, 30]}]
 
If the structure is unknown, return []. Do not invent chapters.`,
      },
    ],
  });
 
  const text = response.content[0].type === 'text' ? response.content[0].text : '[]';
  const match = text.match(/\[[\s\S]*\]/);
  let chapters: Chapter[] = [];
  try {
    chapters = match ? JSON.parse(match[0]) : [];
  } catch {
    chapters = [];
  }
 
  // Persist to Supabase (omitted)
  return c.json({ chapters, source: chapters.length > 0 ? 'ai-estimated' : 'unknown' });
});
 
export default app;

// workers/src/lib/summarize.ts
import Anthropic from '@anthropic-ai/sdk';
 
type SummaryRequest = {
  bookTitle: string;
  chapterTitle: string;
  userNote?: string; // Notes the reader took while reading
  apiKey: string;
};
 
type SummaryResult = {
  keyPoints: string[];
  openQuestions: string[];
  encouragement: string;
};
 
export async function generateChapterSummary(req: SummaryRequest): Promise<SummaryResult> {
  const client = new Anthropic({ apiKey: req.apiKey });
 
  const response = await client.messages.create({
    model: 'claude-sonnet-4-6',
    max_tokens: 1200,
    system: `You are a warm, thoughtful personal reading companion. Your summaries should help the reader consolidate what they just read — without spoiling anything they have not yet reached. Leave space for the reader's own thinking.`,
    messages: [
      {
        role: 'user',
        content: `The reader just finished the chapter "${req.chapterTitle}" of "${req.bookTitle}". Return the following JSON.
 
${req.userNote ? `Reader's note: "${req.userNote}"\n\nReflect this note in the summary.\n` : ''}
 
{
  "keyPoints": ["Three key points, each under 100 characters"],
  "openQuestions": ["Three reflective questions to carry into the next chapter"],
  "encouragement": "A short, non-preachy note for the reader (under 120 characters)"
}
 
Return the JSON only — no preamble, no trailing text.`,
      },
    ],
  });
 
  const text = response.content[0].type === 'text' ? response.content[0].text : '{}';
  const match = text.match(/\{[\s\S]*\}/);
  if (\!match) {
    throw new Error('Summary generation returned no JSON');
  }
 
  try {
    return JSON.parse(match[0]) as SummaryResult;
  } catch (e) {
    throw new Error('Summary JSON parse failed');
  }
}

{
  "keyPoints": [
    "Deep work and shallow work demand different mental modes that rarely coexist in the same day",
    "Cognitive residue from context switching can linger for over 20 minutes after each interruption",
    "Rituals reduce willpower cost by pre-deciding where and when deep work will happen"
  ],
  "openQuestions": [
    "Which parts of my current work genuinely require deep concentration, and which only feel that way?",
    "What small ritual could I commit to this week that protects at least 90 uninterrupted minutes?",
    "Where is shallow work currently disguised as productive deep work in my calendar?"
  ],
  "encouragement": "Let this chapter sit for a day before reading on. Your mind will keep working on it."
}

// lib/storage/bookStorage.ts
import { MMKV } from 'react-native-mmkv';
 
const storage = new MMKV({ id: 'reading-assistant' });
 
type BookState = {
  id: string;
  title: string;
  author?: string;
  currentPage: number;
  totalPages?: number;
  lastReadAt: number;
  pendingSync: boolean;
};
 
export const bookStorage = {
  upsert(book: BookState): void {
    storage.set(`book:${book.id}`, JSON.stringify({ ...book, pendingSync: true }));
  },
 
  get(id: string): BookState | null {
    const raw = storage.getString(`book:${id}`);
    if (\!raw) return null;
    try {
      return JSON.parse(raw);
    } catch {
      return null;
    }
  },
 
  getPendingSync(): BookState[] {
    const keys = storage.getAllKeys().filter((k) => k.startsWith('book:'));
    const books: BookState[] = [];
    for (const key of keys) {
      const raw = storage.getString(key);
      if (\!raw) continue;
      try {
        const book = JSON.parse(raw) as BookState;
        if (book.pendingSync) books.push(book);
      } catch {
        // Ignore corrupted entries and move on
        continue;
      }
    }
    return books;
  },
 
  markSynced(id: string): void {
    const book = this.get(id);
    if (book) {
      storage.set(`book:${id}`, JSON.stringify({ ...book, pendingSync: false }));
    }
  },
};