Rork × Core ML カスタムモデル開発 × オンデバイスAI

// src/ai/inferenceTarget.ts
 
export type InferenceTarget = 'core-ml' | 'cloud';
 
export interface InferenceInput {
  text: string;
  /** 写真・位置情報・連絡先など、端末外に出したくない値を含むか */
  hasSensitivePayload: boolean;
  /** モデルの学習時点より新しい情報を要求しているか */
  needsFreshKnowledge: boolean;
  /** 多段階の推論や長文生成を要求しているか */
  needsDeepReasoning: boolean;
  isOffline: boolean;
}
 
export interface TargetDecision {
  target: InferenceTarget;
  /** ログ・デバッグ用。UI には出さない */
  rationale: string;
}
 
const SHORT_INPUT_LIMIT = 50;
 
export function selectInferenceTarget(input: InferenceInput): TargetDecision {
  // 端末外に出せない条件は、他のどの条件よりも先に評価する
  if (input.isOffline) {
    return { target: 'core-ml', rationale: 'offline' };
  }
  if (input.hasSensitivePayload) {
    return { target: 'core-ml', rationale: 'sensitive-payload' };
  }
 
  // 端末内モデルが原理的に答えられないものだけを外に出す
  if (input.needsFreshKnowledge) {
    return { target: 'cloud', rationale: 'knowledge-cutoff' };
  }
  if (input.needsDeepReasoning) {
    return { target: 'cloud', rationale: 'reasoning-depth' };
  }
 
  // 短い入力は往復コストが割に合わない
  if (input.text.length <= SHORT_INPUT_LIMIT) {
    return { target: 'core-ml', rationale: 'short-input' };
  }
 
  return { target: 'core-ml', rationale: 'default-local-first' };
}

// src/ai/inferenceStats.ts
import AsyncStorage from '@react-native-async-storage/async-storage';
 
const KEY = 'inference_stats_v1';
 
type Bucket = { coreML: number; cloud: number; cloudSpendUsd: number };
 
export async function recordInference(
  target: 'core-ml' | 'cloud',
  spendUsd: number
): Promise<void> {
  const raw = await AsyncStorage.getItem(KEY);
  const bucket: Bucket = raw
    ? JSON.parse(raw)
    : { coreML: 0, cloud: 0, cloudSpendUsd: 0 };
 
  if (target === 'core-ml') {
    bucket.coreML += 1;
  } else {
    bucket.cloud += 1;
    bucket.cloudSpendUsd += spendUsd;
  }
 
  await AsyncStorage.setItem(KEY, JSON.stringify(bucket));
}
 
/** 端末内で処理できた割合。ここが落ちたら振り分け条件を疑う */
export async function localHitRate(): Promise<number> {
  const raw = await AsyncStorage.getItem(KEY);
  if (!raw) return 0;
  const b: Bucket = JSON.parse(raw);
  const total = b.coreML + b.cloud;
  return total === 0 ? 0 : b.coreML / total;
}

// Xcodeメニュー: File > New > ML Model
// UI操作:
// 1. "Image Classification" を選択
// 2. Training Data に dataset/ フォルダを指定
// 3. Validation Data に全体の10-20%を確保
// 4. Output に defect-detection.mlmodel を指定
// 5. "Train" ボタンをクリック

// inspect.swift
import CoreML
 
// 生成クラス経由がいちばん確実です
// （defect-detection.mlmodel → defect_detection クラス）
let wrapper = try defect_detection(configuration: MLModelConfiguration())
print(wrapper.model.modelDescription)
 
// URL から読む場合、バンドルにあるのは .mlmodelc のほうです
guard let url = Bundle.main.url(forResource: "defect-detection", withExtension: "mlmodelc") else {
    throw MLError.modelNotFound
}
let model = try MLModel(contentsOf: url)
print(model.modelDescription)

# pip install coremltools tensorflow
 
import coremltools as ct
import coremltools.optimize as cto
import tensorflow as tf
 
tf_model = tf.keras.models.load_model('defect_model.h5')
 
# 分類器は outputs ではなく classifier_config で宣言します
class_labels = ["ok", "scratch", "dent"]
 
mlmodel = ct.convert(
    tf_model,
    inputs=[ct.ImageType(name="image", shape=(1, 224, 224, 3))],
    classifier_config=ct.ClassifierConfig(class_labels),
    convert_to="mlprogram",
    minimum_deployment_target=ct.target.iOS17,
    compute_units=ct.ComputeUnit.CPU_AND_NE,
)
 
# ML Program の重み量子化はこちらの API です
op_config = cto.coreml.OpLinearQuantizerConfig(
    mode="linear_symmetric", weight_threshold=512
)
config = cto.coreml.OptimizationConfig(global_config=op_config)
quantized_mlmodel = cto.coreml.linear_quantize_weights(mlmodel, config=config)
 
# 保存するのは量子化した側です（ML Program の拡張子は .mlpackage）
quantized_mlmodel.save('DefectDetection.mlpackage')

// 1. MLModel を一度だけ読み込み、必要なときに手放せるようにしておく
actor MLModelCache {
    static let shared = MLModelCache()
    private var model: MLModel?
 
    func model() throws -> MLModel {
        if let model { return model }
        guard let url = Bundle.main.url(forResource: "defect-detection",
                                        withExtension: "mlmodelc") else {
            throw MLError.modelNotFound
        }
        let loaded = try MLModel(contentsOf: url)
        model = loaded
        return loaded
    }
 
    /// メモリ警告を受けたら手放し、次の推論で読み直します
    func release() {
        model = nil
    }
}
 
// 2. 推論時は autoreleasepool で一時メモリを解放
autoreleasepool {
    let prediction = try model.prediction(input: input)
    // prediction を処理
}

// Rork Max が生成するSwiftUIコード (自動)
import CoreML
import Vision
 
struct DefectDetectionView: View {
    @State private var cameraImage: UIImage?
    @State private var detectionResult: String = ""
 
    var body: some View {
        VStack {
            Image(uiImage: cameraImage ?? UIImage())
                .resizable()
                .scaledToFit()
 
            Button("分析") {
                detectDefect()
            }
 
            Text(detectionResult)
                .font(.headline)
                .padding()
        }
    }
 
    private func detectDefect() {
        guard let image = cameraImage else { return }
 
        // Model ロード
        guard let model = try? defect_detection(configuration: MLModelConfiguration()) else {
            detectionResult = "モデル読み込み失敗"
            return
        }
 
        // 入力準備
        guard let buffer = image.pixelBuffer(size: CGSize(width: 224, height: 224)) else {
            return
        }
 
        // 推論実行
        let input = defect_detectionInput(imageAt: buffer)
        guard let output = try? model.prediction(input: input) else {
            detectionResult = "推論失敗"
            return
        }
 
        // 結果表示
        let confidence = output.classLabelProbs.max(by: { $0.value < $1.value })
        detectionResult = "判定: \(confidence?.key ?? "不明") (\(String(format: "%.1f", (confidence?.value ?? 0) * 100))%)"
    }
}

// バッチ推論（複数フレームを連続処理）
func batchInference(frames: [UIImage]) -> [String] {
    let model = try! defect_detection(configuration: MLModelConfiguration())
    var results: [String] = []
 
    for frame in frames {
        guard let buffer = frame.pixelBuffer(size: CGSize(width: 224, height: 224)) else {
            continue
        }
 
        let input = defect_detectionInput(imageAt: buffer)
        let output = try! model.prediction(input: input)
        results.append(output.classLabel)
    }
 
    return results
}

import Vision
import CoreML
 
class ObjectDetector {
    let model: defect_yolo // Core ML モデル
 
    init() throws {
        self.model = try defect_yolo(configuration: MLModelConfiguration())
    }
 
    func detect(in image: UIImage) -> [DetectionResult] {
        guard let ciImage = CIImage(image: image) else { return [] }
 
        // Vision Request 生成
        let request = VNCoreMLRequest(model: try! model.model) { request, error in
            if let error = error {
                print("Detection error: \(error)")
                return
            }
 
            guard let observations = request.results as? [VNRecognizedObjectObservation] else {
                return
            }
 
            // 検出結果の処理
            for observation in observations {
                let boundingBox = observation.boundingBox
                let confidence = observation.confidence
                let label = observation.labels.first?.identifier ?? "Unknown"
 
                print("Detected: \(label) at \(boundingBox) (\(confidence))")
            }
        }
 
        // リクエスト実行
        let handler = VNImageRequestHandler(ciImage: ciImage, options: [:])
        try? handler.perform([request])
 
        return []
    }
}
 
struct DetectionResult {
    let label: String
    let boundingBox: CGRect
    let confidence: Float
}

// Rork が生成するコード
func overlayDetections(on image: CGImage, results: [DetectionResult]) {
    let size = CGSize(width: image.width, height: image.height)
 
    UIGraphicsBeginImageContextWithOptions(size, false, 1.0)
    UIImage(cgImage: image).draw(at: .zero)
 
    let context = UIGraphicsGetCurrentContext()!
    context.setStrokeColor(UIColor.red.cgColor)
    context.setLineWidth(2.0)
 
    for result in results {
        let rect = CGRect(
            x: result.boundingBox.origin.x * size.width,
            y: result.boundingBox.origin.y * size.height,
            width: result.boundingBox.width * size.width,
            height: result.boundingBox.height * size.height
        )
        context.stroke(rect)
 
        // ラベル描画
        let label = NSAttributedString(
            string: "\(result.label) \(String(format: "%.0f%%", result.confidence * 100))",
            attributes: [.foregroundColor: UIColor.red, .font: UIFont.systemFont(ofSize: 12)]
        )
        label.draw(at: CGPoint(x: rect.minX, y: rect.minY - 20))
    }
 
    let annotatedImage = UIGraphicsGetImageFromCurrentImageContext()
    UIGraphicsEndImageContext()
}

import NaturalLanguage
import CoreML
 
class SentimentAnalyzer {
    let model: sentiment_classifier // Core ML
 
    init() throws {
        self.model = try sentiment_classifier(configuration: MLModelConfiguration())
    }
 
    func analyze(_ text: String) -> SentimentResult {
        // テキストの前処理
        let language = NLLanguageRecognizer.dominantLanguage(for: text) ?? .japanese
 
        let tagger = NLTagger(tagSchemes: [.tokenType])
        tagger.string = text
 
        var tokens: [String] = []
        tagger.enumerateTags(in: text.startIndex..<text.endIndex, unit: .word, scheme: .tokenType) { tag, range in
            let token = String(text[range]).lowercased()
            tokens.append(token)
            return true
        }
 
        // モデル入力
        let input = sentiment_classifierInput(text: tokens.joined(separator: " "))
        guard let output = try? model.prediction(input: input) else {
            return SentimentResult(sentiment: "unknown", confidence: 0.0)
        }
 
        return SentimentResult(
            sentiment: output.classLabel,
            confidence: output.classLabelProbs[output.classLabel] ?? 0.0
        )
    }
}
 
struct SentimentResult {
    let sentiment: String // "positive" / "negative"
    let confidence: Double
}

struct ReviewAnalysisView: View {
    @State private var reviewText: String = ""
    @State private var sentiment: SentimentResult?
 
    var body: some View {
        VStack {
            TextEditor(text: $reviewText)
                .border(Color.gray)
 
            Button("分析") {
                analyzeReview()
            }
 
            if let result = sentiment {
                HStack {
                    Image(systemName: result.sentiment == "positive" ? "hand.thumbsup.fill" : "hand.thumbsdown.fill")
                        .foregroundColor(result.sentiment == "positive" ? .green : .red)
 
                    Text(result.sentiment == "positive" ? "肯定的" : "否定的")
                    Text(String(format: "%.0f%%", result.confidence * 100))
                }
                .padding()
                .background(result.sentiment == "positive" ? Color.green.opacity(0.1) : Color.red.opacity(0.1))
                .cornerRadius(8)
            }
        }
    }
 
    private func analyzeReview() {
        let analyzer = try! SentimentAnalyzer()
        sentiment = analyzer.analyze(reviewText)
    }
}

import os.log
 
class PerformanceMonitor {
    let logger = os.log.OSLog(subsystem: "com.rorklab.inference", category: "performance")
 
    func measureInferenceTime<T>(
        _ task: () throws -> T
    ) rethrows -> (result: T, timeMs: Double) {
        let startTime = CACurrentMediaTime()
        let result = try task()
        let endTime = CACurrentMediaTime()
        let timeMs = (endTime - startTime) * 1000
 
        os_log(
            "Inference time: %.1f ms",
            log: logger,
            type: .debug,
            timeMs
        )
 
        return (result, timeMs)
    }
}
 
// 使用例
let monitor = PerformanceMonitor()
let (output, timeMs) = try monitor.measureInferenceTime {
    try model.prediction(input: modelInput)
}
print("推論: \(timeMs)ms")

// 1フレーム: 遅延は小さいが、GPU利用率が低い
for frame in frames {
    let output = try model.prediction(input: frameInput)
}
 
// バッチ処理: 遅延は大きいが、スループット向上
let batchInput = MLMultiArray(shape: [frames.count, 224, 224, 3], dataType: .float32)
// ... フレームをbatchInputに詰める ...
let output = try model.prediction(input: batchInput)

let config = MLModelConfiguration()
 
// CPU のみ（精度最高、速度最低）
config.computeUnits = .cpuOnly
 
// GPU + CPU（バランス型）
config.computeUnits = .cpuAndGPU
 
// Neural Engine + GPU + CPU（最速）
config.computeUnits = .all
 
let model = try defect_detection(configuration: config)

class InferenceQueue {
    private let queue = DispatchQueue(label: "com.rorklab.inference", qos: .userInitiated)
    private let model: defect_detection
 
    init() throws {
        self.model = try defect_detection(configuration: MLModelConfiguration())
    }
 
    func predict(
        _ input: defect_detectionInput,
        completion: @escaping (Result<defect_detectionOutput, Error>) -> Void
    ) {
        queue.async {
            do {
                let output = try self.model.prediction(input: input)
                DispatchQueue.main.async {
                    completion(.success(output))
                }
            } catch {
                DispatchQueue.main.async {
                    completion(.failure(error))
                }
            }
        }
    }
}

import CoreML
 
enum ModelStoreError: Error {
    case bundledModelMissing
    case checksumMismatch
}
 
actor ModelStore {
    private let bundledName = "defect-detection"
    private var cached: MLModel?
 
    /// 常にどれかのモデルを返します。リモートが落ちていても推論は止めません。
    func current(remote: URL, version: String, sha256: String) async -> MLModel? {
        if let cached { return cached }
 
        // 1) すでに導入済みならそれを使う
        if let installed = try? load(version: version) {
            cached = installed
            return installed
        }
 
        // 2) なければ取りに行く
        if let fetched = try? await install(from: remote, version: version, sha256: sha256) {
            cached = fetched
            return fetched
        }
 
        // 3) 何かあれば同梱モデルへ退避する
        cached = try? bundledModel()
        return cached
    }
 
    /// メモリ警告時に呼びます
    func evict() { cached = nil }
 
    private func install(from remote: URL, version: String, sha256: String) async throws -> MLModel {
        // data(from:) はモデル全体をメモリへ載せます。download(from:) を使います
        let (downloaded, response) = try await URLSession.shared.download(from: remote)
        defer { try? FileManager.default.removeItem(at: downloaded) }
 
        guard (response as? HTTPURLResponse)?.statusCode == 200 else {
            throw URLError(.badServerResponse)
        }
        guard try digest(of: downloaded) == sha256.lowercased() else {
            throw ModelStoreError.checksumMismatch
        }
 
        // 端末上でコンパイル。戻り値は一時ディレクトリの .mlmodelc です
        let compiled = try await MLModel.compileModel(at: downloaded)
 
        // 恒久的な場所へ移す。ここを飛ばすと次回起動時には消えています
        let fm = FileManager.default
        let destination = try modelDirectory().appendingPathComponent("\(version).mlmodelc")
        if fm.fileExists(atPath: destination.path) {
            try fm.removeItem(at: destination)
        }
        try fm.moveItem(at: compiled, to: destination)
 
        try? pruneOldVersions(keeping: version)
        return try MLModel(contentsOf: destination)
    }
 
    private func load(version: String) throws -> MLModel {
        let url = try modelDirectory().appendingPathComponent("\(version).mlmodelc")
        guard FileManager.default.fileExists(atPath: url.path) else {
            throw CocoaError(.fileNoSuchFile)
        }
        return try MLModel(contentsOf: url)
    }
 
    private func bundledModel() throws -> MLModel {
        guard let url = Bundle.main.url(forResource: bundledName, withExtension: "mlmodelc") else {
            throw ModelStoreError.bundledModelMissing
        }
        return try MLModel(contentsOf: url)
    }
 
    /// Caches ではなく Application Support に置きます。
    /// Caches はシステムの判断で消され、機内で落ちる原因になりました。
    private func modelDirectory() throws -> URL {
        let fm = FileManager.default
        let base = try fm.url(for: .applicationSupportDirectory, in: .userDomainMask,
                              appropriateFor: nil, create: true)
        let dir = base.appendingPathComponent("ml-models", isDirectory: true)
        try fm.createDirectory(at: dir, withIntermediateDirectories: true)
        return dir
    }
 
    private func pruneOldVersions(keeping version: String) throws {
        let fm = FileManager.default
        let dir = try modelDirectory()
        for item in try fm.contentsOfDirectory(at: dir, includingPropertiesForKeys: nil)
        where item.lastPathComponent != "\(version).mlmodelc" {
            try? fm.removeItem(at: item)
        }
    }
}

import CryptoKit
 
extension ModelStore {
    fileprivate func digest(of url: URL) throws -> String {
        let handle = try FileHandle(forReadingFrom: url)
        defer { try? handle.close() }
 
        var hasher = SHA256()
        while let chunk = try handle.read(upToCount: 1 << 20), !chunk.isEmpty {
            hasher.update(data: chunk)
        }
        return hasher.finalize().map { String(format: "%02x", $0) }.joined()
    }
}

struct ModelManifest: Codable {
    let version: String           // 例: "2026-08-11-a"
    let url: URL
    let sha256: String
    let minimumAppVersion: String // 古いアプリへ新しいモデルを降ろさないため
}

final class SecureMLProcessor {
    private let model: defect_detection
 
    init(model: defect_detection) {
        self.model = model
    }
 
    func classifyWithoutPersisting(_ image: UIImage) throws -> String {
        guard let buffer = image.pixelBuffer(size: CGSize(width: 224, height: 224)) else {
            throw MLError.conversionFailed
        }
 
        // ロックとアンロックは対で書きます。
        // ロックしていないバッファを unlock する呼び出しは、それ自体が誤りです
        CVPixelBufferLockBaseAddress(buffer, .readOnly)
        defer { CVPixelBufferUnlockBaseAddress(buffer, .readOnly) }
 
        let output = try model.prediction(input: defect_detectionInput(imageAt: buffer))
 
        // 結果だけを返し、入力はどこにも書き出しません
        return output.classLabel
    }
 
    /// 自分が作ったサブディレクトリの中身だけを消します。
    /// Caches そのものを removeItem すると、URLCache や画像ライブラリなど
    /// 他の仕組みが使っている領域まで巻き込みます
    func deleteInferenceCache() throws {
        let fm = FileManager.default
        let caches = try fm.url(for: .cachesDirectory, in: .userDomainMask,
                                appropriateFor: nil, create: false)
        let mine = caches.appendingPathComponent("inference", isDirectory: true)
 
        guard fm.fileExists(atPath: mine.path) else { return }
        for item in try fm.contentsOfDirectory(at: mine, includingPropertiesForKeys: nil) {
            try fm.removeItem(at: item)
        }
    }
}
 
enum MLError: Error {
    case conversionFailed
    case modelNotFound
    case privacyPolicyViolation
}

import AVFoundation
 
class CameraPermissionManager {
    static func requestCameraAccess(completion: @escaping (Bool) -> Void) {
        AVCaptureDevice.requestAccess(for: .video) { granted in
            DispatchQueue.main.async {
                completion(granted)
            }
        }
    }
 
    static func requestMicrophoneAccess(completion: @escaping (Bool) -> Void) {
        AVCaptureDevice.requestAccess(for: .audio) { granted in
            DispatchQueue.main.async {
                completion(granted)
            }
        }
    }
}
 
// 使用前のチェック
CameraPermissionManager.requestCameraAccess { granted in
    if granted {
        startCameraSession()
    } else {
        showPermissionAlert()
    }
}

struct PrivacyPolicyCompliance {
    var allowDataSharing = false
    var allowAnalytics = false   // 推論データの分析は行わない
    var dataRetentionDays = 0    // 推論データは保持しない
 
    /// assert は Release ビルドで消えます。
    /// 出荷後も効かせたい確認は guard で書き、throw で止めます
    func validateBeforePrediction() throws {
        guard !allowDataSharing, !allowAnalytics, dataRetentionDays == 0 else {
            throw MLError.privacyPolicyViolation
        }
    }
}