Automating Production Incident Response as a Solo Developer — Crashlytics, Sentry, Slack Routing, and Staged Rollback

// src/observability/sentry-fingerprint.ts
import * as Sentry from "@sentry/react-native";
import crashlytics from "@react-native-firebase/crashlytics";
 
type Severity = "fatal" | "error" | "warning";
 
interface IncidentContext {
  feature: "ads" | "iap" | "auth" | "core";
  buildNumber: string;
  remoteConfigVersion: string;
}
 
/**
 * Force Crashlytics issue clusters and Sentry fingerprints to use the same key,
 * so the same root cause shows up as one ticket in both dashboards.
 */
export function reportIncident(
  err: Error,
  severity: Severity,
  ctx: IncidentContext
): void {
  const cluster = stableClusterKey(err);
 
  // 1. Tag Crashlytics with the cluster key so we can search for it later.
  crashlytics().setAttribute("cluster_key", cluster);
  crashlytics().setAttribute("feature", ctx.feature);
  crashlytics().setAttribute("rc_version", ctx.remoteConfigVersion);
  crashlytics().recordError(err);
 
  // 2. Use the same cluster key as the Sentry fingerprint.
  Sentry.withScope((scope) => {
    scope.setFingerprint([cluster, ctx.feature, ctx.buildNumber]);
    scope.setLevel(severity);
    scope.setTag("feature", ctx.feature);
    scope.setTag("rc_version", ctx.remoteConfigVersion);
    Sentry.captureException(err);
  });
}
 
// Top three frames, with line/column numbers normalised so the cluster
// stays stable across releases that shift unrelated code around.
function stableClusterKey(err: Error): string {
  const frames = (err.stack ?? "")
    .split("\n")
    .slice(1, 4)
    .map((line) => line.replace(/:\d+:\d+/g, ":L:C").trim())
    .join("|");
  return `${err.name}::${frames}`;
}

// functions/src/incident-router.ts
import * as functions from "firebase-functions";
import fetch from "node-fetch";
 
const SLACK_CRITICAL = process.env.SLACK_CRITICAL_WEBHOOK!;
const SLACK_LOG = process.env.SLACK_LOG_WEBHOOK!;
 
interface CrashAlert {
  issue_id: string;
  app_id: string;
  fatal: boolean;
  user_count: number;
  cluster_key?: string;
  feature?: string;
}
 
export const routeCrashAlert = functions
  .region("asia-northeast1")
  .https.onRequest(async (req, res) => {
    const alert = req.body as CrashAlert;
    const target = isCritical(alert) ? SLACK_CRITICAL : SLACK_LOG;
 
    await fetch(target, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({
        text: formatMessage(alert),
        attachments: [
          {
            color: alert.fatal ? "#d33" : "#fc0",
            fields: [
              { title: "feature", value: alert.feature ?? "unknown", short: true },
              { title: "users", value: String(alert.user_count), short: true },
              { title: "cluster", value: alert.cluster_key ?? "—", short: false },
            ],
          },
        ],
      }),
    });
    res.status(204).send();
  });
 
function isCritical(a: CrashAlert): boolean {
  // 30+ affected users on a Fatal, or any Fatal in auth/purchase.
  if (a.fatal && a.user_count >= 30) return true;
  if (a.feature === "auth" || a.feature === "iap") return a.fatal;
  return false;
}
 
function formatMessage(a: CrashAlert): string {
  return `:rotating_light: *${a.feature ?? "unknown"}* — ${a.user_count} users\nissue: \`${a.issue_id}\``;
}

// scripts/rollback.ts — manual CLI for staged rollback
import { GoogleAuth } from "google-auth-library";
 
interface RollbackPlan {
  feature: string;
  // Staged: 100 → 50 → 10 → 0 percent.
  stages: number[];
  pauseSeconds: number;
}
 
async function rollback(plan: RollbackPlan) {
  const auth = new GoogleAuth({
    scopes: "https://www.googleapis.com/auth/firebase.remoteconfig",
  });
  const client = await auth.getClient();
  const projectId = process.env.FIREBASE_PROJECT_ID!;
  const url = `https://firebaseremoteconfig.googleapis.com/v1/projects/${projectId}/remoteConfig`;
 
  for (const stage of plan.stages) {
    console.log(`[rollback] ${plan.feature} -> ${stage}%`);
    const current = await client.request({ url });
    const etag = current.headers["etag"] as string;
    const template = current.data as Record<string, unknown>;
 
    setRolloutPercentage(template, plan.feature, stage);
 
    await client.request({
      url,
      method: "PUT",
      headers: { "If-Match": etag, "Content-Type": "application/json" },
      data: template,
    });
 
    console.log(`[rollback] waiting ${plan.pauseSeconds}s before next stage`);
    await new Promise((r) => setTimeout(r, plan.pauseSeconds * 1000));
  }
}
 
function setRolloutPercentage(
  template: Record<string, unknown>,
  feature: string,
  pct: number
): void {
  // Simplified — actual implementation walks conditions + parameters.
  const params = template["parameters"] as Record<string, { conditionalValues?: Record<string, unknown> }>;
  const entry = params[feature];
  if (!entry?.conditionalValues) throw new Error(`${feature} has no conditions`);
  entry.conditionalValues[`rollout_${pct}`] = { value: "true" };
}
 
rollback({
  feature: "feature_new_paywall",
  stages: [100, 50, 10, 0],
  pauseSeconds: 60,
});

# YYYY-MM-DD <feature> incident
 
## What happened
- Detection timestamp / recovery timestamp / estimated affected users.
 
## Why it happened
- Direct cause / root cause / whether this class of incident has occurred before.
 
## What would prevent recurrence
- Code change / monitoring added / docs added — issue link each.
 
## Note to future self
- One line.