Rork × Core ML カスタムモデル開発 × オンデバイスAI

// src/ai/inferenceTarget.ts
 
export type InferenceTarget = 'core-ml' | 'cloud';
 
export interface InferenceInput {
  text: string;
  /** 写真・位置情報・連絡先など、端末外に出したくない値を含むか */
  hasSensitivePayload: boolean;
  /** モデルの学習時点より新しい情報を要求しているか */
  needsFreshKnowledge: boolean;
  /** 多段階の推論や長文生成を要求しているか */
  needsDeepReasoning: boolean;
  isOffline: boolean;
}
 
export interface TargetDecision {
  target: InferenceTarget;
  /** ログ・デバッグ用。UI には出さない */
  rationale: string;
}
 
const SHORT_INPUT_LIMIT = 50;
 
export function selectInferenceTarget(input: InferenceInput): TargetDecision {
  // 端末外に出せない条件は、他のどの条件よりも先に評価する
  if (input.isOffline) {
    return { target: 'core-ml', rationale: 'offline' };
  }
  if (input.hasSensitivePayload) {
    return { target: 'core-ml', rationale: 'sensitive-payload' };
  }
 
  // 端末内モデルが原理的に答えられないものだけを外に出す
  if (input.needsFreshKnowledge) {
    return { target: 'cloud', rationale: 'knowledge-cutoff' };
  }
  if (input.needsDeepReasoning) {
    return { target: 'cloud', rationale: 'reasoning-depth' };
  }
 
  // 短い入力は往復コストが割に合わない
  if (input.text.length <= SHORT_INPUT_LIMIT) {
    return { target: 'core-ml', rationale: 'short-input' };
  }
 
  return { target: 'core-ml', rationale: 'default-local-first' };
}

// src/ai/inferenceStats.ts
import AsyncStorage from '@react-native-async-storage/async-storage';
 
const KEY = 'inference_stats_v1';
 
type Bucket = { coreML: number; cloud: number; cloudSpendUsd: number };
 
export async function recordInference(
  target: 'core-ml' | 'cloud',
  spendUsd: number
): Promise<void> {
  const raw = await AsyncStorage.getItem(KEY);
  const bucket: Bucket = raw
    ? JSON.parse(raw)
    : { coreML: 0, cloud: 0, cloudSpendUsd: 0 };
 
  if (target === 'core-ml') {
    bucket.coreML += 1;
  } else {
    bucket.cloud += 1;
    bucket.cloudSpendUsd += spendUsd;
  }
 
  await AsyncStorage.setItem(KEY, JSON.stringify(bucket));
}
 
/** 端末内で処理できた割合。ここが落ちたら振り分け条件を疑う */
export async function localHitRate(): Promise<number> {
  const raw = await AsyncStorage.getItem(KEY);
  if (!raw) return 0;
  const b: Bucket = JSON.parse(raw);
  const total = b.coreML + b.cloud;
  return total === 0 ? 0 : b.coreML / total;
}

// Xcodeメニュー: File > New > ML Model
// UI操作:
// 1. "Image Classification" を選択
// 2. Training Data に dataset/ フォルダを指定
// 3. Validation Data に全体の10-20%を確保
// 4. Output に defect-detection.mlmodel を指定
// 5. "Train" ボタンをクリック

// Xcodeでモデルメタデータを確認
// inspect.swift
import CoreML
 
let modelURL = Bundle.main.url(forResource: "defect-detection", withExtension: "mlmodel")!
let model = try MLModel(contentsOf: modelURL)
print("Model Size: \(model.modelDescription)")

# インストール
pip install coremltools
 
# Appleが提供するスクリプト例
import coremltools as ct
from coremltools.models import neural_network
 
# TensorFlow モデルを読み込み
tf_model = tf.keras.models.load_model('defect_model.h5')
 
# Core ML に変換
mlmodel = ct.convert(
    tf_model,
    inputs=[ct.ImageType(name="image", shape=(1, 224, 224, 3))],
    outputs=[ct.ClassifierOutputType(name="defects")],
    compute_units=ct.ComputeUnit.CPU_AND_NE  # CPU + Neural Engine
)
 
# Int8量子化を適用
quantized_mlmodel = ct.quantization_utils.quantize_weights(
    mlmodel,
    nbits=8,
    quantization_mode="linear"
)
 
# 保存
mlmodel.save('defect-detection.mlmodel')

// 1. MLModel を一度だけ読み込み、キャッシュ
class MLModelCache {
    static let shared = MLModelCache()
    private let model: MLModel
 
    private init() {
        let url = Bundle.main.url(forResource: "defect-detection", withExtension: "mlmodel")!
        self.model = try! MLModel(contentsOf: url)
    }
}
 
// 2. 推論時は autoreleasepool で一時メモリを解放
autoreleasepool {
    let prediction = try model.prediction(input: input)
    // prediction を処理
}

// Rork Max が生成するSwiftUIコード (自動)
import CoreML
import Vision
 
struct DefectDetectionView: View {
    @State private var cameraImage: UIImage?
    @State private var detectionResult: String = ""
 
    var body: some View {
        VStack {
            Image(uiImage: cameraImage ?? UIImage())
                .resizable()
                .scaledToFit()
 
            Button("分析") {
                detectDefect()
            }
 
            Text(detectionResult)
                .font(.headline)
                .padding()
        }
    }
 
    private func detectDefect() {
        guard let image = cameraImage else { return }
 
        // Model ロード
        guard let model = try? defect_detection(configuration: MLModelConfiguration()) else {
            detectionResult = "モデル読み込み失敗"
            return
        }
 
        // 入力準備
        guard let buffer = image.pixelBuffer(size: CGSize(width: 224, height: 224)) else {
            return
        }
 
        // 推論実行
        let input = defect_detectionInput(imageAt: buffer)
        guard let output = try? model.prediction(input: input) else {
            detectionResult = "推論失敗"
            return
        }
 
        // 結果表示
        let confidence = output.classLabelProbs.max(by: { $0.value < $1.value })
        detectionResult = "判定: \(confidence?.key ?? "不明") (\(String(format: "%.1f", (confidence?.value ?? 0) * 100))%)"
    }
}

// バッチ推論（複数フレームを連続処理）
func batchInference(frames: [UIImage]) -> [String] {
    let model = try! defect_detection(configuration: MLModelConfiguration())
    var results: [String] = []
 
    for frame in frames {
        guard let buffer = frame.pixelBuffer(size: CGSize(width: 224, height: 224)) else {
            continue
        }
 
        let input = defect_detectionInput(imageAt: buffer)
        let output = try! model.prediction(input: input)
        results.append(output.classLabel)
    }
 
    return results
}

import Vision
import CoreML
 
class ObjectDetector {
    let model: defect_yolo // Core ML モデル
 
    init() throws {
        self.model = try defect_yolo(configuration: MLModelConfiguration())
    }
 
    func detect(in image: UIImage) -> [DetectionResult] {
        guard let ciImage = CIImage(image: image) else { return [] }
 
        // Vision Request 生成
        let request = VNCoreMLRequest(model: try! model.model) { request, error in
            if let error = error {
                print("Detection error: \(error)")
                return
            }
 
            guard let observations = request.results as? [VNRecognizedObjectObservation] else {
                return
            }
 
            // 検出結果の処理
            for observation in observations {
                let boundingBox = observation.boundingBox
                let confidence = observation.confidence
                let label = observation.labels.first?.identifier ?? "Unknown"
 
                print("Detected: \(label) at \(boundingBox) (\(confidence))")
            }
        }
 
        // リクエスト実行
        let handler = VNImageRequestHandler(ciImage: ciImage, options: [:])
        try? handler.perform([request])
 
        return []
    }
}
 
struct DetectionResult {
    let label: String
    let boundingBox: CGRect
    let confidence: Float
}

// Rork が生成するコード
func overlayDetections(on image: CGImage, results: [DetectionResult]) {
    let size = CGSize(width: image.width, height: image.height)
 
    UIGraphicsBeginImageContextWithOptions(size, false, 1.0)
    UIImage(cgImage: image).draw(at: .zero)
 
    let context = UIGraphicsGetCurrentContext()!
    context.setStrokeColor(UIColor.red.cgColor)
    context.setLineWidth(2.0)
 
    for result in results {
        let rect = CGRect(
            x: result.boundingBox.origin.x * size.width,
            y: result.boundingBox.origin.y * size.height,
            width: result.boundingBox.width * size.width,
            height: result.boundingBox.height * size.height
        )
        context.stroke(rect)
 
        // ラベル描画
        let label = NSAttributedString(
            string: "\(result.label) \(String(format: "%.0f%%", result.confidence * 100))",
            attributes: [.foregroundColor: UIColor.red, .font: UIFont.systemFont(ofSize: 12)]
        )
        label.draw(at: CGPoint(x: rect.minX, y: rect.minY - 20))
    }
 
    let annotatedImage = UIGraphicsGetImageFromCurrentImageContext()
    UIGraphicsEndImageContext()
}

import NaturalLanguage
import CoreML
 
class SentimentAnalyzer {
    let model: sentiment_classifier // Core ML
 
    init() throws {
        self.model = try sentiment_classifier(configuration: MLModelConfiguration())
    }
 
    func analyze(_ text: String) -> SentimentResult {
        // テキストの前処理
        let language = NLLanguageRecognizer.dominantLanguage(for: text) ?? .japanese
 
        let tagger = NLTagger(tagSchemes: [.tokenType])
        tagger.string = text
 
        var tokens: [String] = []
        tagger.enumerateTags(in: text.startIndex..<text.endIndex, unit: .word, scheme: .tokenType) { tag, range in
            let token = String(text[range]).lowercased()
            tokens.append(token)
            return true
        }
 
        // モデル入力
        let input = sentiment_classifierInput(text: tokens.joined(separator: " "))
        guard let output = try? model.prediction(input: input) else {
            return SentimentResult(sentiment: "unknown", confidence: 0.0)
        }
 
        return SentimentResult(
            sentiment: output.classLabel,
            confidence: output.classLabelProbs[output.classLabel] ?? 0.0
        )
    }
}
 
struct SentimentResult {
    let sentiment: String // "positive" / "negative"
    let confidence: Double
}

struct ReviewAnalysisView: View {
    @State private var reviewText: String = ""
    @State private var sentiment: SentimentResult?
 
    var body: some View {
        VStack {
            TextEditor(text: $reviewText)
                .border(Color.gray)
 
            Button("分析") {
                analyzeReview()
            }
 
            if let result = sentiment {
                HStack {
                    Image(systemName: result.sentiment == "positive" ? "hand.thumbsup.fill" : "hand.thumbsdown.fill")
                        .foregroundColor(result.sentiment == "positive" ? .green : .red)
 
                    Text(result.sentiment == "positive" ? "肯定的" : "否定的")
                    Text(String(format: "%.0f%%", result.confidence * 100))
                }
                .padding()
                .background(result.sentiment == "positive" ? Color.green.opacity(0.1) : Color.red.opacity(0.1))
                .cornerRadius(8)
            }
        }
    }
 
    private func analyzeReview() {
        let analyzer = try! SentimentAnalyzer()
        sentiment = analyzer.analyze(reviewText)
    }
}

import os.log
 
class PerformanceMonitor {
    let logger = os.log.OSLog(subsystem: "com.rorklab.inference", category: "performance")
 
    func measureInferenceTime<T>(
        _ task: () throws -> T
    ) rethrows -> (result: T, timeMs: Double) {
        let startTime = CACurrentMediaTime()
        let result = try task()
        let endTime = CACurrentMediaTime()
        let timeMs = (endTime - startTime) * 1000
 
        os_log(
            "Inference time: %.1f ms",
            log: logger,
            type: .debug,
            timeMs
        )
 
        return (result, timeMs)
    }
}
 
// 使用例
let monitor = PerformanceMonitor()
let (output, timeMs) = try monitor.measureInferenceTime {
    try model.prediction(input: modelInput)
}
print("推論: \(timeMs)ms")

// 1フレーム: 遅延は小さいが、GPU利用率が低い
for frame in frames {
    let output = try model.prediction(input: frameInput)
}
 
// バッチ処理: 遅延は大きいが、スループット向上
let batchInput = MLMultiArray(shape: [frames.count, 224, 224, 3], dataType: .float32)
// ... フレームをbatchInputに詰める ...
let output = try model.prediction(input: batchInput)

let config = MLModelConfiguration()
 
// CPU のみ（精度最高、速度最低）
config.computeUnits = .cpuOnly
 
// GPU + CPU（バランス型）
config.computeUnits = .cpuAndGPU
 
// Neural Engine + GPU + CPU（最速）
config.computeUnits = .all
 
let model = try defect_detection(configuration: config)

class InferenceQueue {
    private let queue = DispatchQueue(label: "com.rorklab.inference", qos: .userInitiated)
    private let model: defect_detection
 
    init() throws {
        self.model = try defect_detection(configuration: MLModelConfiguration())
    }
 
    func predict(
        _ input: defect_detectionInput,
        completion: @escaping (Result<defect_detectionOutput, Error>) -> Void
    ) {
        queue.async {
            do {
                let output = try self.model.prediction(input: input)
                DispatchQueue.main.async {
                    completion(.success(output))
                }
            } catch {
                DispatchQueue.main.async {
                    completion(.failure(error))
                }
            }
        }
    }
}

struct ModelManager {
    let remoteURL = URL(string: "https://api.rorklab.com/models/defect-detection-v2.mlmodel")!
    let localCachePath = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0]
        .appendingPathComponent("models")
 
    func fetchLatestModel() async throws -> MLModel {
        // ローカルキャッシュを確認
        let versionFile = localCachePath.appendingPathComponent("version.json")
        if FileManager.default.fileExists(atPath: versionFile.path) {
            let data = try Data(contentsOf: versionFile)
            let version = try JSONDecoder().decode(ModelVersion.self, from: data)
 
            if version.id == "v2" {
                let modelPath = localCachePath.appendingPathComponent("defect-detection.mlmodel")
                return try MLModel(contentsOf: modelPath)
            }
        }
 
        // リモートから最新モデルを取得
        let (data, _) = try await URLSession.shared.data(from: remoteURL)
 
        // キャッシュに保存
        try FileManager.default.createDirectory(at: localCachePath, withIntermediateDirectories: true)
        let modelPath = localCachePath.appendingPathComponent("defect-detection.mlmodel")
        try data.write(to: modelPath)
 
        // バージョン記録
        let newVersion = ModelVersion(id: "v2", timestamp: Date())
        let versionData = try JSONEncoder().encode(newVersion)
        try versionData.write(to: versionFile)
 
        return try MLModel(contentsOf: modelPath)
    }
}
 
struct ModelVersion: Codable {
    let id: String
    let timestamp: Date
}

enum ModelVersionControl {
    case bundled    // アプリに同梱（フォールバック用）
    case remote(URL) // リモートサーバー（最新）
 
    static func load() async throws -> MLModel {
        do {
            // まずリモートから取得を試みる
            switch Self.remote(URL(string: "https://api.rorklab.com/models/latest.mlmodel")!) {
            case .remote(let url):
                return try await ModelManager().fetch(from: url)
            case .bundled:
                return try bundledModel()
            }
        } catch {
            // ネットワークエラー時はフォールバック
            print("Remote load failed, using bundled model: \(error)")
            return try bundledModel()
        }
    }
 
    private static func bundledModel() throws -> MLModel {
        let url = Bundle.main.url(forResource: "defect-detection", withExtension: "mlmodel")!
        return try MLModel(contentsOf: url)
    }
}

class SecureMLProcessor {
    func processImageWithoutPersisting(_ image: UIImage) throws -> String {
        var processedBuffer: CVPixelBuffer?
 
        defer {
            // 推論後、メモリを即座にクリア
            if var buffer = processedBuffer {
                CVPixelBufferUnlockBaseAddress(buffer, .readOnly)
                processedBuffer = nil
            }
        }
 
        // UIImage → CVPixelBuffer（推論用）
        guard let buffer = image.pixelBuffer(size: CGSize(width: 224, height: 224)) else {
            throw MLError.conversionFailed
        }
        processedBuffer = buffer
 
        // 推論実行
        let input = defect_detectionInput(imageAt: buffer)
        let output = try model.prediction(input: input)
 
        // 結果をメモリのみで返す（ファイル保存しない）
        return output.classLabel
    }
 
    func deleteAllLocalData() throws {
        let cachePath = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0]
        try FileManager.default.removeItem(at: cachePath)
    }
}
 
enum MLError: Error {
    case conversionFailed
}

import AVFoundation
 
class CameraPermissionManager {
    static func requestCameraAccess(completion: @escaping (Bool) -> Void) {
        AVCaptureDevice.requestAccess(for: .video) { granted in
            DispatchQueue.main.async {
                completion(granted)
            }
        }
    }
 
    static func requestMicrophoneAccess(completion: @escaping (Bool) -> Void) {
        AVCaptureDevice.requestAccess(for: .audio) { granted in
            DispatchQueue.main.async {
                completion(granted)
            }
        }
    }
}
 
// 使用前のチェック
CameraPermissionManager.requestCameraAccess { granted in
    if granted {
        startCameraSession()
    } else {
        showPermissionAlert()
    }
}

struct PrivacyPolicyCompliance {
    // プライバシー設定
    let allowDataSharing: Bool = false
    let allowAnalytics: Bool = false // 推論データ分析禁止
    let dataRetentionDays: Int = 0     // データ保持しない
 
    func validateBeforePrediction() -> Bool {
        // サーバーへの送信がないことを確認
        assert(!allowDataSharing, "Data sharing must be disabled")
        assert(!allowAnalytics, "Analytics must be disabled")
        return true
    }
}