Getting a 1GB Video Out of a Rork App Without Losing It — TUS, S3 Multipart, and Background Transfers

// store/uploadStore.ts
import { create } from 'zustand';
import * as FileSystem from 'expo-file-system';
import AsyncStorage from '@react-native-async-storage/async-storage';
 
type Status = 'idle' | 'uploading' | 'paused' | 'failed' | 'completed';
 
export interface UploadTask {
  id: string;
  fileUri: string;
  fileName: string;
  fileSize: number;
  mimeType: string;
  chunkSize: number;           // in bytes (e.g. 5 * 1024 * 1024)
  uploadedBytes: number;       // offset when resuming
  status: Status;
  serverUploadUrl?: string;    // TUS URL / Multipart uploadId
  errorMessage?: string;
  retryCount: number;
}
 
interface UploadState {
  tasks: Record<string, UploadTask>;
  addTask: (task: UploadTask) => Promise<void>;
  updateProgress: (id: string, uploadedBytes: number) => void;
  setStatus: (id: string, status: Status, error?: string) => void;
  hydrateFromStorage: () => Promise<void>;
}
 
const STORAGE_KEY = 'rork-upload-queue-v1';
 
export const useUploadStore = create<UploadState>((set, get) => ({
  tasks: {},
 
  addTask: async (task) => {
    set((s) => ({ tasks: { ...s.tasks, [task.id]: task } }));
    await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(get().tasks));
  },
 
  updateProgress: (id, uploadedBytes) => {
    const task = get().tasks[id];
    if (!task) return;
    // Persist every 1MB to keep write frequency sane
    const shouldPersist = uploadedBytes - task.uploadedBytes >= 1024 * 1024;
    set((s) => ({
      tasks: { ...s.tasks, [id]: { ...task, uploadedBytes } },
    }));
    if (shouldPersist) {
      AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(get().tasks));
    }
  },
 
  setStatus: (id, status, error) => {
    const task = get().tasks[id];
    if (!task) return;
    set((s) => ({
      tasks: {
        ...s.tasks,
        [id]: { ...task, status, errorMessage: error },
      },
    }));
    AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(get().tasks));
  },
 
  hydrateFromStorage: async () => {
    const raw = await AsyncStorage.getItem(STORAGE_KEY);
    if (!raw) return;
    try {
      const tasks = JSON.parse(raw) as Record<string, UploadTask>;
      // Any task left in "uploading" gets demoted to "paused" on rehydrate
      Object.values(tasks).forEach((t) => {
        if (t.status === 'uploading') t.status = 'paused';
      });
      set({ tasks });
    } catch (e) {
      console.warn('upload-store hydrate failed', e);
    }
  },
}));

// lib/uploadTus.ts
import * as tus from 'tus-js-client';
import * as FileSystem from 'expo-file-system';
import { useUploadStore } from '../store/uploadStore';
 
// Upload to a TUS server in chunks.
// Expected behavior: 5MB POST/PATCH per chunk; can resume after disconnection.
export async function uploadWithTus(
  taskId: string,
  endpoint: string,
  fileUri: string,
  fileName: string,
  mimeType: string,
): Promise<void> {
  const store = useUploadStore.getState();
  const task = store.tasks[taskId];
  if (!task) throw new Error(`task not found: ${taskId}`);
 
  // Expo cannot hand Blob directly — read the file as a stream
  const fileInfo = await FileSystem.getInfoAsync(fileUri);
  if (!fileInfo.exists) throw new Error('file missing');
 
  return new Promise((resolve, reject) => {
    const upload = new tus.Upload(
      // Lightweight wrapper for React Native
      {
        uri: fileUri,
        name: fileName,
        type: mimeType,
        size: fileInfo.size,
      } as any,
      {
        endpoint,
        chunkSize: 5 * 1024 * 1024, // 5MB
        retryDelays: [0, 1000, 3000, 5000, 10000], // exponential backoff
        metadata: { filename: fileName, filetype: mimeType },
        // Progress is driven by the server's Upload-Offset response
        onProgress: (bytesUploaded) => {
          store.updateProgress(taskId, bytesUploaded);
        },
        onSuccess: () => {
          store.setStatus(taskId, 'completed');
          resolve();
        },
        onError: (err) => {
          store.setStatus(taskId, 'failed', err.message);
          reject(err);
        },
        // Resume from the previously issued upload URL if present
        uploadUrl: task.serverUploadUrl,
      },
    );
 
    // On first run, the server issues an upload URL we persist for resume
    upload.findPreviousUploads().then((previousUploads) => {
      if (previousUploads.length > 0) {
        upload.resumeFromPreviousUpload(previousUploads[0]);
      }
      store.setStatus(taskId, 'uploading');
      upload.start();
    });
  });
}

// lib/uploadMultipart.ts
import * as FileSystem from 'expo-file-system';
import { useUploadStore } from '../store/uploadStore';
 
interface InitiateResponse {
  uploadId: string;
  key: string;
}
 
interface PartUrl {
  partNumber: number;
  url: string;
}
 
const PART_SIZE = 5 * 1024 * 1024; // R2/S3 minimum is 5MB
const PARALLELISM = 3;              // 3 is the sweet spot on cellular
 
// Coordinates with your API to run a multipart upload.
// Expected behavior: 1GB file splits into 200 parts, uploaded 3-at-a-time.
export async function uploadWithMultipart(
  taskId: string,
  fileUri: string,
  fileName: string,
  apiBaseUrl: string,
  authToken: string,
): Promise<void> {
  const store = useUploadStore.getState();
  const task = store.tasks[taskId];
  if (!task) throw new Error('task not found');
 
  const fileInfo = await FileSystem.getInfoAsync(fileUri);
  if (!fileInfo.exists) throw new Error('file missing');
  const totalSize = fileInfo.size;
  const totalParts = Math.ceil(totalSize / PART_SIZE);
 
  try {
    store.setStatus(taskId, 'uploading');
 
    // Step 1: initiate multipart
    const initRes = await fetch(`${apiBaseUrl}/uploads/initiate`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        Authorization: `Bearer ${authToken}`,
      },
      body: JSON.stringify({ fileName, size: totalSize, totalParts }),
    });
    if (!initRes.ok) throw new Error(`initiate failed: ${initRes.status}`);
    const init = (await initRes.json()) as InitiateResponse;
 
    // Persist uploadId for resume on next run
    store.setStatus(taskId, 'uploading');
 
    // Step 2: get pre-signed URLs for every part
    const partUrlsRes = await fetch(`${apiBaseUrl}/uploads/part-urls`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        Authorization: `Bearer ${authToken}`,
      },
      body: JSON.stringify({
        uploadId: init.uploadId,
        key: init.key,
        totalParts,
      }),
    });
    const { parts } = (await partUrlsRes.json()) as { parts: PartUrl[] };
 
    // Step 3: upload in parallel
    const completedParts: { PartNumber: number; ETag: string }[] = [];
    let uploadedBytes = task.uploadedBytes || 0;
 
    async function uploadPart(part: PartUrl): Promise<void> {
      const start = (part.partNumber - 1) * PART_SIZE;
      const end = Math.min(start + PART_SIZE, totalSize);
      const partSize = end - start;
 
      // Read only the byte range for this part
      const base64 = await FileSystem.readAsStringAsync(fileUri, {
        encoding: FileSystem.EncodingType.Base64,
        position: start,
        length: partSize,
      });
      const binary = decodeBase64ToUint8Array(base64);
 
      const res = await fetch(part.url, {
        method: 'PUT',
        body: binary as any,
      });
      if (!res.ok) throw new Error(`part ${part.partNumber} failed`);
 
      const etag = res.headers.get('ETag')?.replace(/"/g, '') || '';
      completedParts.push({ PartNumber: part.partNumber, ETag: etag });
 
      uploadedBytes += partSize;
      store.updateProgress(taskId, uploadedBytes);
    }
 
    // Bounded concurrency across all parts
    const queue = [...parts];
    const workers = Array.from({ length: PARALLELISM }, async () => {
      while (queue.length > 0) {
        const next = queue.shift();
        if (!next) return;
        await retryWithBackoff(() => uploadPart(next), 3);
      }
    });
    await Promise.all(workers);
 
    // Step 4: complete
    completedParts.sort((a, b) => a.PartNumber - b.PartNumber);
    const completeRes = await fetch(`${apiBaseUrl}/uploads/complete`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        Authorization: `Bearer ${authToken}`,
      },
      body: JSON.stringify({
        uploadId: init.uploadId,
        key: init.key,
        parts: completedParts,
      }),
    });
    if (!completeRes.ok) throw new Error('complete failed');
 
    store.setStatus(taskId, 'completed');
  } catch (e: any) {
    store.setStatus(taskId, 'failed', e.message);
    throw e;
  }
}
 
function decodeBase64ToUint8Array(base64: string): Uint8Array {
  const binString = atob(base64);
  const bytes = new Uint8Array(binString.length);
  for (let i = 0; i < binString.length; i++) bytes[i] = binString.charCodeAt(i);
  return bytes;
}
 
async function retryWithBackoff<T>(fn: () => Promise<T>, max: number): Promise<T> {
  let lastErr: any;
  for (let i = 0; i < max; i++) {
    try {
      return await fn();
    } catch (e) {
      lastErr = e;
      await new Promise((r) => setTimeout(r, 500 * Math.pow(2, i)));
    }
  }
  throw lastErr;
}

// lib/backgroundUpload.ts
import * as BackgroundFetch from 'expo-background-fetch';
import * as TaskManager from 'expo-task-manager';
import { useUploadStore } from '../store/uploadStore';
import { uploadWithMultipart } from './uploadMultipart';
 
const TASK_NAME = 'rork-resumable-upload';
 
// Background handler picks up any unfinished task and resumes it.
// Expected behavior: iOS calls defineTask when it deems it a good time.
TaskManager.defineTask(TASK_NAME, async () => {
  try {
    await useUploadStore.getState().hydrateFromStorage();
    const tasks = useUploadStore.getState().tasks;
    const pending = Object.values(tasks).filter(
      (t) => t.status === 'paused' || t.status === 'failed',
    );
 
    for (const task of pending) {
      try {
        await uploadWithMultipart(
          task.id,
          task.fileUri,
          task.fileName,
          process.env.EXPO_PUBLIC_API_URL!,
          await getAuthToken(),
        );
      } catch (e) {
        console.warn('bg upload failed', task.id, e);
      }
    }
    return BackgroundFetch.BackgroundFetchResult.NewData;
  } catch (e) {
    return BackgroundFetch.BackgroundFetchResult.Failed;
  }
});
 
export async function registerBackgroundUpload(): Promise<void> {
  await BackgroundFetch.registerTaskAsync(TASK_NAME, {
    minimumInterval: 60 * 15, // 15 min (iOS ignores smaller values)
    stopOnTerminate: false,
    startOnBoot: true,
  });
}
 
async function getAuthToken(): Promise<string> {
  // Omitted: pull JWT from expo-secure-store
  return '';
}

// components/UploadProgressList.tsx
import React from 'react';
import { View, Text, Pressable, StyleSheet } from 'react-native';
import { useUploadStore, UploadTask } from '../store/uploadStore';
 
export function UploadProgressList() {
  const tasks = useUploadStore((s) => Object.values(s.tasks));
 
  if (tasks.length === 0) return null;
 
  return (
    <View style={styles.container}>
      {tasks.map((task) => (
        <UploadRow key={task.id} task={task} />
      ))}
    </View>
  );
}
 
function UploadRow({ task }: { task: UploadTask }) {
  const percent = Math.min(100, (task.uploadedBytes / task.fileSize) * 100);
  const setStatus = useUploadStore((s) => s.setStatus);
 
  const statusLabel = {
    idle: 'Queued',
    uploading: `Uploading ${percent.toFixed(0)}%`,
    paused: 'Paused — tap to resume',
    failed: `Failed — ${task.errorMessage ?? ''}`,
    completed: 'Done',
  }[task.status];
 
  return (
    <Pressable
      onPress={() => {
        if (task.status === 'paused' || task.status === 'failed') {
          setStatus(task.id, 'uploading');
        }
      }}
      style={styles.row}
    >
      <Text style={styles.fileName} numberOfLines={1}>
        {task.fileName}
      </Text>
      <View style={styles.barBg}>
        <View style={[styles.barFill, { width: `${percent}%` }]} />
      </View>
      <Text style={styles.status}>{statusLabel}</Text>
    </Pressable>
  );
}
 
const styles = StyleSheet.create({
  container: { padding: 16, gap: 12 },
  row: { gap: 6 },
  fileName: { fontSize: 14, fontWeight: '600' },
  barBg: { height: 6, backgroundColor: '#eee', borderRadius: 3 },
  barFill: { height: 6, backgroundColor: '#4F7BFF', borderRadius: 3 },
  status: { fontSize: 12, color: '#666' },
});

// api/uploads.ts (Cloudflare Workers + Hono)
import { Hono } from 'hono';
import { AwsClient } from 'aws4fetch';
 
type Env = {
  R2_BUCKET: R2Bucket;
  R2_ACCOUNT_ID: string;
  R2_ACCESS_KEY_ID: string;
  R2_SECRET_ACCESS_KEY: string;
  R2_BUCKET_NAME: string;
};
 
const app = new Hono<{ Bindings: Env }>();
 
// 1. Initiate a multipart upload. Returns uploadId + object key.
app.post('/uploads/initiate', async (c) => {
  const { fileName, size, totalParts } = await c.req.json<{
    fileName: string;
    size: number;
    totalParts: number;
  }>();
  if (size > 5 * 1024 * 1024 * 1024) return c.json({ error: 'too large' }, 400);
 
  const key = `u/${crypto.randomUUID()}/${fileName}`;
  const multipart = await c.env.R2_BUCKET.createMultipartUpload(key);
  return c.json({ uploadId: multipart.uploadId, key });
});
 
// 2. Mint pre-signed URLs for each part (valid 1 hour).
app.post('/uploads/part-urls', async (c) => {
  const { uploadId, key, totalParts } = await c.req.json<{
    uploadId: string;
    key: string;
    totalParts: number;
  }>();
 
  const aws = new AwsClient({
    accessKeyId: c.env.R2_ACCESS_KEY_ID,
    secretAccessKey: c.env.R2_SECRET_ACCESS_KEY,
    service: 's3',
    region: 'auto',
  });
 
  const endpoint = `https://${c.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`;
  const parts: { partNumber: number; url: string }[] = [];
  for (let i = 1; i <= totalParts; i++) {
    const url = `${endpoint}/${c.env.R2_BUCKET_NAME}/${encodeURIComponent(key)}?partNumber=${i}&uploadId=${uploadId}`;
    const signed = await aws.sign(url, {
      method: 'PUT',
      aws: { signQuery: true, expiresIn: 60 * 60 },
    });
    parts.push({ partNumber: i, url: signed.url });
  }
  return c.json({ parts });
});
 
// 3. Complete the multipart upload once all parts are done.
app.post('/uploads/complete', async (c) => {
  const { uploadId, key, parts } = await c.req.json<{
    uploadId: string;
    key: string;
    parts: { PartNumber: number; ETag: string }[];
  }>();
  const multipart = c.env.R2_BUCKET.resumeMultipartUpload(key, uploadId);
  const object = await multipart.complete(
    parts.map((p) => ({ partNumber: p.PartNumber, etag: p.ETag })),
  );
  return c.json({ ok: true, key: object.key, size: object.size });
});
 
// 4. Abort an in-flight upload (client cancel path).
app.post('/uploads/abort', async (c) => {
  const { uploadId, key } = await c.req.json<{ uploadId: string; key: string }>();
  const multipart = c.env.R2_BUCKET.resumeMultipartUpload(key, uploadId);
  await multipart.abort();
  return c.json({ ok: true });
});
 
export default app;