Rork × AIモデレーション実装:UGCアプリで通報・自動審査・段階的制裁を本番投入する

// workers/src/post-router.ts
import { Hono } from "hono";
 
type Env = {
  POSTS_KV: KVNamespace;
  MODERATION_QUEUE: Queue<ModerationJob>;
  PERSPECTIVE_API_KEY: string;
  OPENAI_API_KEY: string;
};
 
type ModerationJob = {
  postId: string;
  userId: string;
  text: string;
  imageUrls: string[];
  createdAt: number;
};
 
const app = new Hono<{ Bindings: Env }>();
 
app.post("/api/posts", async (c) => {
  const { text, imageUrls = [] } = await c.req.json();
  const userId = c.req.header("X-User-Id");
  if (!userId) {
    return c.json({ error: "Unauthorized" }, 401);
  }
 
  // サーバー側でも改めてpreflightを実行(クライアント改ざん対策)
  if (text.trim().length < 2 || text.length > 1000) {
    return c.json({ error: "Invalid length" }, 400);
  }
 
  const postId = crypto.randomUUID();
  const post = {
    id: postId,
    userId,
    text,
    imageUrls,
    status: "pending_moderation" as const,
    createdAt: Date.now(),
  };
 
  // 投稿はpendingでKVに即時保存し、ユーザーには202を返す
  await c.env.POSTS_KV.put(`post:${postId}`, JSON.stringify(post), {
    expirationTtl: 60 * 60 * 24 * 30, // 30日でクリーンアップ対象
  });
 
  // モデレーションをQueueに非同期enqueue
  await c.env.MODERATION_QUEUE.send({
    postId,
    userId,
    text,
    imageUrls,
    createdAt: post.createdAt,
  });
 
  return c.json({ postId, status: "pending_moderation" }, 202);
});
 
app.get("/api/posts/:id", async (c) => {
  const id = c.req.param("id");
  const post = await c.env.POSTS_KV.get(`post:${id}`, "json");
  if (!post) return c.json({ error: "Not found" }, 404);
  return c.json(post);
});
 
export default app;

// workers/src/moderation-consumer.ts
type Verdict = "approved" | "rejected" | "needs_review";
 
async function checkText(text: string, env: Env): Promise<Verdict> {
  // ① Perspective APIで第一選別
  const perspective = await fetch(
    `https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze?key=${env.PERSPECTIVE_API_KEY}`,
    {
      method: "POST",
      body: JSON.stringify({
        comment: { text },
        requestedAttributes: {
          TOXICITY: {},
          SEVERE_TOXICITY: {},
          THREAT: {},
          SEXUALLY_EXPLICIT: {},
        },
        languages: ["ja", "en"],
      }),
    },
  ).then((r) => r.json() as Promise<PerspectiveResponse>);
 
  const scores = perspective.attributeScores;
  const severe = scores.SEVERE_TOXICITY.summaryScore.value;
  const threat = scores.THREAT.summaryScore.value;
 
  if (severe > 0.8 || threat > 0.7) return "rejected";
  if (severe < 0.3 && threat < 0.3) {
    // 明らかに安全な投稿はOpenAIをスキップしてコスト削減
    return "approved";
  }
 
  // ② グレーゾーンはOpenAIで再評価
  const openai = await fetch("https://api.openai.com/v1/moderations", {
    method: "POST",
    headers: {
      Authorization: `Bearer ${env.OPENAI_API_KEY}`,
      "Content-Type": "application/json",
    },
    body: JSON.stringify({ model: "omni-moderation-latest", input: text }),
  }).then((r) => r.json() as Promise<OpenAIModerationResponse>);
 
  const result = openai.results[0];
  if (result.flagged) {
    // 自傷示唆・未成年保護関連は例外なくrejected
    if (
      result.categories["self-harm/intent"] ||
      result.categories["self-harm/instructions"] ||
      result.categories["sexual/minors"]
    ) {
      return "rejected";
    }
    // それ以外のflaggedはneeds_review(人間レビュー)
    return "needs_review";
  }
  return "approved";
}
 
export default {
  async queue(batch: MessageBatch<ModerationJob>, env: Env) {
    for (const message of batch.messages) {
      const job = message.body;
      try {
        const textVerdict = await checkText(job.text, env);
 
        const post = (await env.POSTS_KV.get(`post:${job.postId}`, "json")) as Post | null;
        if (!post) {
          message.ack();
          continue;
        }
 
        post.status =
          textVerdict === "approved"
            ? "published"
            : textVerdict === "rejected"
              ? "rejected"
              : "needs_review";
        post.moderatedAt = Date.now();
        post.moderationVerdict = textVerdict;
 
        await env.POSTS_KV.put(`post:${job.postId}`, JSON.stringify(post));
        message.ack();
      } catch (err) {
        // 失敗時はretryさせる(maxRetries到達時はDLQへ)
        message.retry({ delaySeconds: 30 });
      }
    }
  },
};

// workers/src/report-handler.ts
app.post("/api/posts/:id/report", async (c) => {
  const postId = c.req.param("id");
  const reporterId = c.req.header("X-User-Id");
  const { reason, detail } = await c.req.json();
 
  if (!reporterId || !reason) {
    return c.json({ error: "Bad request" }, 400);
  }
 
  const reportId = crypto.randomUUID();
  await c.env.REPORTS_KV.put(
    `report:${postId}:${reporterId}`,
    JSON.stringify({
      reportId,
      postId,
      reporterId,
      reason,
      detail: detail?.slice(0, 500),
      createdAt: Date.now(),
    }),
  );
 
  // 通報したユーザーから当該投稿を即座にmute(自動非表示)
  const muteKey = `mute:${reporterId}:${postId}`;
  await c.env.MUTES_KV.put(muteKey, "1");
 
  // 同一投稿への通報数が閾値を超えたら自動的にneeds_reviewへ移行
  const reportCount = await incrementReportCounter(c.env, postId);
  if (reportCount >= 3) {
    const post = (await c.env.POSTS_KV.get(`post:${postId}`, "json")) as Post | null;
    if (post && post.status === "published") {
      post.status = "needs_review";
      post.flaggedReason = "auto_threshold";
      await c.env.POSTS_KV.put(`post:${postId}`, JSON.stringify(post));
    }
  }
 
  return c.json({ ok: true, reportId });
});
 
async function incrementReportCounter(env: Env, postId: string): Promise<number> {
  const key = `report_count:${postId}`;
  const current = parseInt((await env.COUNTERS_KV.get(key)) || "0", 10);
  const next = current + 1;
  await env.COUNTERS_KV.put(key, String(next), { expirationTtl: 60 * 60 * 24 * 7 });
  return next;
}