Đánh giá tính năng LLM: biết nó chạy đúng và giữ cho nó đúng

export type Expectations = {
  mustInclude?: string[];
  mustExclude?: string[];
  maxWords?: number;
  expectedCategory?: "billing" | "bug" | "account" | "other";
  rubric?: string;
};

export type EvalCase<Input> = {
  id: string;
  input: Input;
  tags: string[];
  expect: Expectations;
};

{
  "id": "ticket-0142",
  "tags": ["mixed-language", "long-input", "from-bug-report"],
  "input": { "ticketText": "Hi team, mình bị trừ tiền hai lần cho order tháng trước..." },
  "expect": {
    "mustInclude": ["hoàn tiền"],
    "mustExclude": ["@gmail.com"],
    "maxWords": 60,
    "expectedCategory": "billing",
    "rubric": "The summary states what the customer is asking for and does not invent order numbers, amounts or dates."
  }
}

import { z } from "zod";

export type CheckResult = { name: string; passed: boolean; detail?: string };

export const ticketSummarySchema = z.object({
  summary: z.string().min(1),
  category: z.enum(["billing", "bug", "account", "other"]),
  urgency: z.enum(["low", "normal", "high"]),
});

export type TicketSummary = z.infer<typeof ticketSummarySchema>;

export function checkSchema(raw: string): { result: CheckResult; value?: TicketSummary } {
  let json: unknown;
  try {
    json = JSON.parse(raw);
  } catch {
    return { result: { name: "schema", passed: false, detail: "not valid JSON" } };
  }
  const parsed = ticketSummarySchema.safeParse(json);
  if (!parsed.success) {
    const detail = parsed.error.issues.map((issue) => `${issue.path.join(".")}: ${issue.message}`).join("; ");
    return { result: { name: "schema", passed: false, detail } };
  }
  return { result: { name: "schema", passed: true }, value: parsed.data };
}

export function checkContent(value: TicketSummary, expect: Expectations): CheckResult[] {
  const text = value.summary.toLowerCase();
  const results: CheckResult[] = [];
  for (const phrase of expect.mustInclude ?? []) {
    results.push({ name: `includes:${phrase}`, passed: text.includes(phrase.toLowerCase()) });
  }
  for (const phrase of expect.mustExclude ?? []) {
    results.push({ name: `excludes:${phrase}`, passed: !text.includes(phrase.toLowerCase()) });
  }
  if (expect.maxWords !== undefined) {
    const words = value.summary.trim().split(/\s+/).length;
    results.push({ name: "length", passed: words <= expect.maxWords, detail: `${words} words` });
  }
  if (expect.expectedCategory) {
    results.push({ name: "category", passed: value.category === expect.expectedCategory, detail: value.category });
  }
  return results;
}

type Complete = (request: { system: string; user: string; temperature?: number }) => Promise<string>;

const verdictSchema = z.object({
  reasoning: z.string(),
  verdict: z.enum(["pass", "fail"]),
});

export async function judgeWithRubric(complete: Complete, rubric: string, input: string, output: string): Promise<CheckResult> {
  const raw = await complete({
    temperature: 0,
    system: [
      "You grade one output of an AI feature against a single criterion.",
      "Judge only that criterion. Length and style do not matter unless the criterion mentions them.",
      'Reply with JSON only: {"reasoning": string, "verdict": "pass" | "fail"}. Write the reasoning first.',
    ].join("\n"),
    user: `Criterion:\n${rubric}\n\nInput:\n${input}\n\nOutput to grade:\n${output}`,
  });
  try {
    const { reasoning, verdict } = verdictSchema.parse(JSON.parse(raw));
    return { name: "judge", passed: verdict === "pass", detail: reasoning };
  } catch {
    return { name: "judge", passed: false, detail: `unparseable judge reply: ${raw.slice(0, 200)}` };
  }
}

import { completeWithJudgeModel } from "@/lib/ai";

export async function gradeTicketSummary(
  testCase: EvalCase<{ ticketText: string }>,
  output: string,
): Promise<CheckResult[]> {
  const { result, value } = checkSchema(output);
  if (!value) return [result];
  const checks = [result, ...checkContent(value, testCase.expect)];
  if (testCase.expect.rubric) {
    checks.push(await judgeWithRubric(completeWithJudgeModel, testCase.expect.rubric, testCase.input.ticketText, value.summary));
  }
  return checks;
}

type Preference = "A" | "B" | "tie";

async function askPreference(
  complete: Complete,
  rubric: string,
  input: string,
  responseA: string,
  responseB: string,
): Promise<Preference> {
  const raw = await complete({
    temperature: 0,
    system: `Compare two responses using this rubric:\n${rubric}\nReply with exactly one token: A, B or tie.`,
    user: `Input:\n${input}\n\nResponse A:\n${responseA}\n\nResponse B:\n${responseB}`,
  });
  const answer = raw.trim();
  return answer === "A" || answer === "B" ? answer : "tie";
}

export async function comparePairwise(complete: Complete, rubric: string, input: string, baseline: string, candidate: string) {
  const [forward, backward] = await Promise.all([
    askPreference(complete, rubric, input, baseline, candidate),
    askPreference(complete, rubric, input, candidate, baseline),
  ]);
  const forwardWinner = forward === "A" ? "baseline" : forward === "B" ? "candidate" : "tie";
  const backwardWinner = backward === "A" ? "candidate" : backward === "B" ? "baseline" : "tie";
  return forwardWinner === backwardWinner ? forwardWinner : "tie";
}

export type Generation = { output: string; inputTokens: number; outputTokens: number; latencyMs: number };

export type Variant<Input> = {
  name: string;
  generate: (input: Input) => Promise<Generation>;
  usdPerMillionTokens: { input: number; output: number };
};

type CaseRun = { caseId: string; passed: boolean; checks: CheckResult[]; generation?: Generation };

async function mapWithConcurrency<T, R>(items: T[], limit: number, worker: (item: T) => Promise<R>): Promise<R[]> {
  const results = new Array<R>(items.length);
  let cursor = 0;
  async function lane() {
    while (cursor < items.length) {
      const index = cursor++;
      results[index] = await worker(items[index]);
    }
  }
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, lane));
  return results;
}

function percentile(values: number[], p: number): number {
  if (values.length === 0) return 0;
  const sorted = [...values].sort((left, right) => left - right);
  return sorted[Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)];
}

export async function runEval<Input>({
  cases,
  variant,
  grade,
  repeats = 3,
  concurrency = 8,
}: {
  cases: EvalCase<Input>[];
  variant: Variant<Input>;
  grade: (testCase: EvalCase<Input>, output: string) => Promise<CheckResult[]>;
  repeats?: number;
  concurrency?: number;
}) {
  const jobs = cases.flatMap((testCase) => Array.from({ length: repeats }, () => testCase));
  const runs = await mapWithConcurrency(jobs, concurrency, async (testCase): Promise<CaseRun> => {
    try {
      const generation = await variant.generate(testCase.input);
      const checks = await grade(testCase, generation.output);
      return { caseId: testCase.id, passed: checks.every((check) => check.passed), checks, generation };
    } catch (error) {
      return { caseId: testCase.id, passed: false, checks: [{ name: "generate", passed: false, detail: String(error) }] };
    }
  });

  const generations = runs.flatMap((run) => (run.generation ? [run.generation] : []));
  const { input: inputPrice, output: outputPrice } = variant.usdPerMillionTokens;
  const passRateByCase = Object.fromEntries(
    cases.map((testCase) => {
      const caseRuns = runs.filter((run) => run.caseId === testCase.id);
      return [testCase.id, caseRuns.filter((run) => run.passed).length / caseRuns.length];
    }),
  );

  return {
    variant: variant.name,
    passRate: runs.filter((run) => run.passed).length / runs.length,
    passRateByCase,
    flakyCaseIds: Object.keys(passRateByCase).filter((id) => passRateByCase[id] > 0 && passRateByCase[id] < 1),
    costUsd: generations.reduce((total, g) => total + (g.inputTokens * inputPrice + g.outputTokens * outputPrice) / 1_000_000, 0),
    latencyMs: {
      p50: percentile(generations.map((g) => g.latencyMs), 50),
      p95: percentile(generations.map((g) => g.latencyMs), 95),
    },
    failures: runs.filter((run) => !run.passed),
  };
}

export type EvalReport = Awaited<ReturnType<typeof runEval>>;

export function findRegressions(baseline: EvalReport, current: EvalReport, tolerance = 0.03) {
  const brokenCaseIds = Object.keys(current.passRateByCase).filter(
    (id) => baseline.passRateByCase[id] === 1 && current.passRateByCase[id] === 0,
  );
  const overallDrop = baseline.passRate - current.passRate;
  return { brokenCaseIds, overallDrop, failed: brokenCaseIds.length > 0 || overallDrop > tolerance };
}

import { readFile, writeFile } from "node:fs/promises";
import { cases } from "./cases";
import { currentVariant } from "./variants";
import { gradeTicketSummary } from "./grade";
import { findRegressions, runEval, type EvalReport } from "./runner";

const report = await runEval({ cases, variant: currentVariant, grade: gradeTicketSummary });
const baseline: EvalReport = JSON.parse(await readFile("evals/baseline.json", "utf8"));
const { brokenCaseIds, overallDrop, failed } = findRegressions(baseline, report);

await writeFile("evals/report.json", JSON.stringify(report, null, 2));
console.log(
  `pass ${(report.passRate * 100).toFixed(1)}% (drop ${(overallDrop * 100).toFixed(1)} pts), ` +
    `cost $${report.costUsd.toFixed(2)}, p95 ${report.latencyMs.p95}ms`,
);
if (brokenCaseIds.length > 0) console.log(`broken: ${brokenCaseIds.join(", ")}`);
process.exit(failed ? 1 : 0);

export type LlmTrace = {
  traceId: string;
  feature: string;
  promptVersion: string;
  model: string;
  input: string;
  retrievedIds?: string[];
  output: string;
  inputTokens: number;
  outputTokens: number;
  latencyMs: number;
  timeToFirstTokenMs?: number;
  error?: string;
};

export type FeedbackEvent =
  | { traceId: string; kind: "thumb"; value: "up" | "down"; comment?: string }
  | { traceId: string; kind: "edited"; changedCharRatio: number }
  | { traceId: string; kind: "regenerated" };