Rork + Langfuse で本番 AI アプリのトレース・コスト・品質を計測する運用ガイド

// worker/src/routes/chat.ts
// Rork アプリから呼ばれる /api/chat エンドポイント
// ユーザー発話を受け取り → OpenAI に投げ → Langfuse にトレースを残す
import { Hono } from "hono";
import { Langfuse } from "langfuse";
import OpenAI from "openai";
 
type Env = {
  OPENAI_API_KEY: string;
  LANGFUSE_PUBLIC_KEY: string;
  LANGFUSE_SECRET_KEY: string;
  LANGFUSE_BASE_URL: string; // 例: https://cloud.langfuse.com
};
 
export const chatRoute = new Hono<{ Bindings: Env }>();
 
chatRoute.post("/chat", async (c) => {
  const { userId, message, sessionId } = await c.req.json<{
    userId: string;
    sessionId: string;
    message: string;
  }>();
 
  const langfuse = new Langfuse({
    publicKey: c.env.LANGFUSE_PUBLIC_KEY,
    secretKey: c.env.LANGFUSE_SECRET_KEY,
    baseUrl: c.env.LANGFUSE_BASE_URL,
  });
 
  // トレース単位 = 「1回のユーザー発話に対する応答サイクル」
  const trace = langfuse.trace({
    name: "chat-turn",
    userId,
    sessionId,
    input: { message },
    tags: ["production", "chat-v2"],
  });
 
  const openai = new OpenAI({ apiKey: c.env.OPENAI_API_KEY });
 
  // 生成ステップを Langfuse の generation として記録
  const generation = trace.generation({
    name: "openai-chat",
    model: "gpt-4.1-mini",
    input: [{ role: "user", content: message }],
  });
 
  try {
    const response = await openai.chat.completions.create({
      model: "gpt-4.1-mini",
      messages: [{ role: "user", content: message }],
    });
 
    const output = response.choices[0].message.content ?? "";
 
    generation.end({
      output,
      usage: {
        promptTokens: response.usage?.prompt_tokens,
        completionTokens: response.usage?.completion_tokens,
        totalTokens: response.usage?.total_tokens,
      },
    });
 
    trace.update({ output: { reply: output } });
 
    // Workers は終了するとバックグラウンドタスクが切られるので flush 必須
    await langfuse.flushAsync();
 
    return c.json({ reply: output, traceId: trace.id });
  } catch (error) {
    // 失敗も記録する — 本番では失敗トレースこそ宝
    generation.end({
      level: "ERROR",
      statusMessage: error instanceof Error ? error.message : "unknown",
    });
    await langfuse.flushAsync();
    return c.json({ error: "Upstream LLM error" }, 502);
  }
});

// worker/src/crons/cost-alert.ts
// 毎日 9:00 JST に前日分の LLM コスト合計を計算し、
// 閾値を超えていたら Slack と Rork アプリのプッシュ通知で管理者に通知
import { Langfuse } from "langfuse";
 
type Env = {
  LANGFUSE_PUBLIC_KEY: string;
  LANGFUSE_SECRET_KEY: string;
  LANGFUSE_BASE_URL: string;
  SLACK_WEBHOOK_URL: string;
  DAILY_COST_THRESHOLD_USD: string; // 例: "50"
};
 
export async function runCostAlert(env: Env): Promise<void> {
  const langfuse = new Langfuse({
    publicKey: env.LANGFUSE_PUBLIC_KEY,
    secretKey: env.LANGFUSE_SECRET_KEY,
    baseUrl: env.LANGFUSE_BASE_URL,
  });
 
  const now = new Date();
  const start = new Date(now.getTime() - 24 * 60 * 60 * 1000);
 
  try {
    // Langfuse Public API: /api/public/metrics/daily
    const res = await fetch(
      `${env.LANGFUSE_BASE_URL}/api/public/metrics/daily?fromTimestamp=${start.toISOString()}&toTimestamp=${now.toISOString()}`,
      {
        headers: {
          Authorization:
            "Basic " +
            btoa(`${env.LANGFUSE_PUBLIC_KEY}:${env.LANGFUSE_SECRET_KEY}`),
        },
      },
    );
 
    if (!res.ok) throw new Error(`Langfuse API ${res.status}`);
    const json = (await res.json()) as {
      data: { totalCost: number; totalTokens: number }[];
    };
 
    const totalCostUSD = json.data.reduce((s, d) => s + d.totalCost, 0);
    const threshold = Number(env.DAILY_COST_THRESHOLD_USD);
 
    if (totalCostUSD >= threshold) {
      await fetch(env.SLACK_WEBHOOK_URL, {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({
          text: `⚠️ Rork AI cost alert — yesterday: $${totalCostUSD.toFixed(2)} (threshold $${threshold})`,
        }),
      });
    }
  } catch (err) {
    // 通知側の失敗でクロン自体を止めない
    console.error("cost-alert failed:", err);
  } finally {
    await langfuse.flushAsync();
  }
}

// worker/src/crons/eval-chat-quality.ts
// 毎日 2:00 JST に前日のチャットトレースを抽出し、
// LLM-as-a-judge で「意図理解スコア」を付与する
import { Langfuse } from "langfuse";
import OpenAI from "openai";
 
const JUDGE_PROMPT = `
あなたはカスタマーサポート品質評価の専門家です。
以下のユーザー発話とアシスタント応答を読み、次の2軸を 1〜5 で採点してください。
- intent_understanding: ユーザーの意図を正しく捉えているか
- actionability: ユーザーが次に何をすべきか明確か
 
JSON 形式で必ず以下を返してください:
{"intent_understanding": 1-5, "actionability": 1-5, "reason": "..."}
`;
 
export async function runQualityEval(env: {
  LANGFUSE_PUBLIC_KEY: string;
  LANGFUSE_SECRET_KEY: string;
  LANGFUSE_BASE_URL: string;
  OPENAI_API_KEY: string;
}): Promise<void> {
  const langfuse = new Langfuse({
    publicKey: env.LANGFUSE_PUBLIC_KEY,
    secretKey: env.LANGFUSE_SECRET_KEY,
    baseUrl: env.LANGFUSE_BASE_URL,
  });
  const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY });
 
  const since = new Date(Date.now() - 24 * 60 * 60 * 1000).toISOString();
 
  // Langfuse から対象トレースを取得（簡略化した fetch 例）
  const res = await fetch(
    `${env.LANGFUSE_BASE_URL}/api/public/traces?name=chat-turn&fromTimestamp=${since}&limit=100`,
    {
      headers: {
        Authorization:
          "Basic " +
          btoa(`${env.LANGFUSE_PUBLIC_KEY}:${env.LANGFUSE_SECRET_KEY}`),
      },
    },
  );
  if (!res.ok) throw new Error(`list traces ${res.status}`);
  const { data } = (await res.json()) as {
    data: { id: string; input: { message?: string }; output: { reply?: string } }[];
  };
 
  for (const trace of data) {
    const userMsg = trace.input?.message ?? "";
    const reply = trace.output?.reply ?? "";
    if (!userMsg || !reply) continue;
 
    try {
      const judge = await openai.chat.completions.create({
        model: "gpt-4.1-mini",
        response_format: { type: "json_object" },
        messages: [
          { role: "system", content: JUDGE_PROMPT },
          { role: "user", content: `USER: ${userMsg}\nASSISTANT: ${reply}` },
        ],
      });
 
      const parsed = JSON.parse(judge.choices[0].message.content ?? "{}") as {
        intent_understanding?: number;
        actionability?: number;
        reason?: string;
      };
 
      // スコアを該当トレースに書き戻す
      if (typeof parsed.intent_understanding === "number") {
        langfuse.score({
          traceId: trace.id,
          name: "intent_understanding",
          value: parsed.intent_understanding,
          comment: parsed.reason,
        });
      }
      if (typeof parsed.actionability === "number") {
        langfuse.score({
          traceId: trace.id,
          name: "actionability",
          value: parsed.actionability,
        });
      }
    } catch (err) {
      // 1件の失敗で全体を止めない
      console.error("judge failed for trace", trace.id, err);
    }
  }
 
  await langfuse.flushAsync();
}

// app/components/ChatFeedback.tsx
// メッセージの下に表示する「役に立った？」ボタン
import { useState } from "react";
import { Pressable, Text, View, Alert } from "react-native";
 
type Props = { traceId: string };
 
export function ChatFeedback({ traceId }: Props) {
  const [sent, setSent] = useState<"up" | "down" | null>(null);
 
  async function send(value: 1 | -1) {
    try {
      const res = await fetch(`${process.env.EXPO_PUBLIC_API_BASE}/chat/feedback`, {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ traceId, value }),
      });
      if (!res.ok) throw new Error(`HTTP ${res.status}`);
      setSent(value === 1 ? "up" : "down");
    } catch (err) {
      Alert.alert("送信できませんでした", "通信状況を確認してもう一度お試しください。");
    }
  }
 
  if (sent) {
    return (
      <View><Text>ご協力ありがとうございました</Text></View>
    );
  }
 
  return (
    <View style={{ flexDirection: "row", gap: 12 }}>
      <Pressable onPress={() => send(1)} accessibilityLabel="役に立った">
        <Text>👍</Text>
      </Pressable>
      <Pressable onPress={() => send(-1)} accessibilityLabel="役に立たなかった">
        <Text>👎</Text>
      </Pressable>
    </View>
  );
}