Rork で『AI が自動でアルバム化する写真整理アプリ』を作る実装ガイド — Vision × CLIP × pgvector で本番品質のパイプラインを設計する

import Vision
 
struct PhotoAnalysisResult {
  let assetIdentifier: String
  let labels: [String]               // 物体ラベル（"dog", "beach" 等）
  let texts: [String]                // OCR で抽出したテキスト
  let faceCount: Int                 // 顔の数
  let saliencyBox: CGRect?           // 主要被写体のバウンディングボックス
  let visionFeaturePrint: Data       // Vision の特徴量
  let aestheticScore: Float          // 写真の品質スコア（iOS 18+）
}
 
func analyzeWithVision(image: CGImage, identifier: String) async throws -> PhotoAnalysisResult {
  let handler = VNImageRequestHandler(cgImage: image, options: [:])
 
  // 1. 物体分類
  let classifyRequest = VNClassifyImageRequest()
  classifyRequest.usesCPUOnly = false  // Neural Engine を使う
 
  // 2. OCR
  let ocrRequest = VNRecognizeTextRequest()
  ocrRequest.recognitionLevel = .accurate
  ocrRequest.recognitionLanguages = ["ja-JP", "en-US"]
 
  // 3. 顔検出
  let faceRequest = VNDetectFaceRectanglesRequest()
 
  // 4. 特徴量
  let featurePrintRequest = VNGenerateImageFeaturePrintRequest()
 
  try handler.perform([classifyRequest, ocrRequest, faceRequest, featurePrintRequest])
 
  let labels = (classifyRequest.results ?? [])
    .filter { $0.confidence > 0.5 }
    .prefix(10)
    .map { $0.identifier }
 
  let texts = (ocrRequest.results ?? [])
    .compactMap { $0.topCandidates(1).first?.string }
    .filter { !$0.isEmpty }
 
  let faceCount = faceRequest.results?.count ?? 0
 
  guard let featurePrint = featurePrintRequest.results?.first as? VNFeaturePrintObservation else {
    throw NSError(domain: "Vision", code: -1)
  }
 
  return PhotoAnalysisResult(
    assetIdentifier: identifier,
    labels: Array(labels),
    texts: texts,
    faceCount: faceCount,
    saliencyBox: nil,
    visionFeaturePrint: featurePrint.data,
    aestheticScore: 0.0
  )
}

import CoreML
 
class CLIPEmbeddingService {
  private let model: MobileCLIPS1
  private let imageProcessor: VNCoreMLRequest
 
  init() throws {
    let config = MLModelConfiguration()
    config.computeUnits = .cpuAndNeuralEngine  // Neural Engine を優先
    self.model = try MobileCLIPS1(configuration: config)
    let coreMLModel = try VNCoreMLModel(for: model.model)
    self.imageProcessor = VNCoreMLRequest(model: coreMLModel)
    // ⚠️ .centerCrop は写真検索では選ばないでください（後述の実測を参照）
    self.imageProcessor.imageCropAndScaleOption = .scaleFill
  }
 
  func embedImage(_ cgImage: CGImage) async throws -> [Float] {
    let handler = VNImageRequestHandler(cgImage: cgImage, options: [:])
    try handler.perform([imageProcessor])
    guard let result = imageProcessor.results?.first as? VNCoreMLFeatureValueObservation,
          let array = result.featureValue.multiArrayValue else {
      throw NSError(domain: "CLIP", code: -1)
    }
    // 512 次元の Float32 ベクトルに変換
    let count = array.count
    var embedding = [Float](repeating: 0, count: count)
    for i in 0..<count {
      embedding[i] = array[i].floatValue
    }
    // L2 正規化（pgvector でコサイン類似度を使うため）
    let norm = sqrt(embedding.reduce(0) { $0 + $1 * $1 })
    guard norm > 1e-6 else { throw NSError(domain: "CLIP", code: -2) }  // 全黒画像などで 0 になります
    return embedding.map { $0 / norm }
  }
 
  func embedText(_ text: String) async throws -> [Float] {
    // CLIP の BPE で 77 トークン固定長へ。前後に SOT / EOT を入れ、残りは 0 で埋めます
    let ids = tokenizer.encode(text, contextLength: 77)
    let input = try MLMultiArray(shape: [1, 77], dataType: .int32)
    for (i, id) in ids.enumerated() { input[i] = NSNumber(value: Int32(id)) }
 
    let out = try textModel.prediction(input_text: input)
    guard let array = out.featureValue(for: "text_embeddings")?.multiArrayValue else {
      throw NSError(domain: "CLIP", code: -3)
    }
    var embedding = [Float](repeating: 0, count: array.count)
    for i in 0..<array.count { embedding[i] = array[i].floatValue }
 
    // 画像側とまったく同じ正規化を通します。ここが揃わないと内積が意味を失います
    let norm = sqrt(embedding.reduce(0) { $0 + $1 * $1 })
    guard norm > 1e-6 else { throw NSError(domain: "CLIP", code: -2) }
    return embedding.map { $0 / norm }
  }
}

-- pgvector 拡張を有効化
create extension if not exists vector;
 
-- 画像メタデータテーブル
create table photo_assets (
  id uuid primary key default gen_random_uuid(),
  user_id uuid references auth.users(id) on delete cascade,
  asset_identifier text not null,         -- iOS の PHAsset.localIdentifier
  taken_at timestamptz,
  labels text[],                          -- Vision の物体ラベル
  ocr_text text,                          -- OCR 全文（GIN インデックス用）
  face_count int default 0,
  embedding vector(512),                  -- CLIP 埋め込み
  created_at timestamptz default now(),
  unique(user_id, asset_identifier)
);
 
-- HNSW インデックス（コサイン類似度）
create index on photo_assets
  using hnsw (embedding vector_cosine_ops)
  with (m = 16, ef_construction = 64);
 
-- ラベル検索用 GIN インデックス
create index on photo_assets using gin (labels);
 
-- OCR 全文検索（日本語）
create index on photo_assets using gin (to_tsvector('simple', ocr_text));
 
-- RLS 設定
alter table photo_assets enable row level security;
create policy "users can read own assets"
  on photo_assets for select
  using (auth.uid() = user_id);
create policy "users can insert own assets"
  on photo_assets for insert
  with check (auth.uid() = user_id);

-- 自然言語クエリ + メタデータフィルタのハイブリッド検索
create or replace function search_photos_hybrid(
  query_embedding vector(512),
  filter_labels text[],
  match_count int default 30
)
returns table (
  id uuid,
  asset_identifier text,
  labels text[],
  similarity float
)
language sql stable as $$
  select
    p.id,
    p.asset_identifier,
    p.labels,
    1 - (p.embedding <=> query_embedding) as similarity
  from photo_assets p
  where p.user_id = auth.uid()
    and (filter_labels is null or p.labels && filter_labels)
  order by p.embedding <=> query_embedding
  limit match_count;
$$;

create or replace function search_photos_hybrid(
  query_embedding vector(512),
  filter_labels text[],
  match_count int default 30
)
returns table (
  id uuid,
  asset_identifier text,
  labels text[],
  similarity float
)
language plpgsql stable as $$
begin
  -- 絞り込みが効いた状態でも match_count 件を満たすまでグラフを辿ります
  set local hnsw.iterative_scan = 'relaxed_order';
  set local hnsw.max_scan_tuples = 20000;   -- 打ち切り上限（既定値）
  set local hnsw.ef_search = 100;           -- 1 回あたりの候補数。既定は 40
 
  return query
    select
      p.id,
      p.asset_identifier,
      p.labels,
      1 - (p.embedding <=> query_embedding) as similarity
    from photo_assets p
    where p.user_id = auth.uid()
      and (filter_labels is null or p.labels && filter_labels)
    order by p.embedding <=> query_embedding
    limit match_count;
end;
$$;

import BackgroundTasks
 
let backgroundTaskIdentifier = "com.example.photoorganizer.indexing"
 
func registerBackgroundTask() {
  BGTaskScheduler.shared.register(
    forTaskWithIdentifier: backgroundTaskIdentifier,
    using: nil
  ) { task in
    handleBackgroundIndexing(task: task as! BGProcessingTask)
  }
}
 
func scheduleBackgroundIndexing() {
  let request = BGProcessingTaskRequest(identifier: backgroundTaskIdentifier)
  request.requiresNetworkConnectivity = true   // pgvector 同期のため
  request.requiresExternalPower = true          // CLIP は電池を食う
  request.earliestBeginDate = Date(timeIntervalSinceNow: 60 * 60)  // 1 時間後
 
  do {
    try BGTaskScheduler.shared.submit(request)
  } catch {
    print("BGTask scheduling failed: \(error)")
  }
}
 
func handleBackgroundIndexing(task: BGProcessingTask) {
  scheduleBackgroundIndexing()  // 次回分を予約
 
  let queue = OperationQueue()
  queue.maxConcurrentOperationCount = 2  // Neural Engine の競合回避
 
  task.expirationHandler = {
    queue.cancelAllOperations()
  }
 
  Task {
    do {
      let pending = try await fetchPendingAssets(limit: 500)
      for asset in pending {
        guard !task.isCancelled else { break }
        try await processAsset(asset)
      }
      task.setTaskCompleted(success: true)
    } catch {
      task.setTaskCompleted(success: false)
    }
  }
}

// React Native 側 — Supabase Edge Function を呼ぶ
async function generateSmartAlbums() {
  const response = await fetch(
    `${SUPABASE_URL}/functions/v1/generate-albums`,
    {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${session.access_token}`,
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({ cluster_count: 50 }),
    }
  );
  const albums = await response.json();
  // [{ id, title, cover_asset_identifier, asset_count }, ...]
  return albums;
}