14.17-RAG评估指标

function recallAtK(
  retrieved: string[],
  relevant: string[],
  k: number
): number {
  const topK = retrieved.slice(0, k)
  const found = topK.filter((id) => relevant.includes(id)).length
  return found / relevant.length
}

function mrr(queries: Array<{ retrieved: string[]; relevant: string[] }>): number {
  let sum = 0
 
  for (const { retrieved, relevant } of queries) {
    const firstCorrect = retrieved.findIndex((id) => relevant.includes(id))
    if (firstCorrect >= 0) {
      sum += 1 / (firstCorrect + 1)  // rank 从 1 开始
    }
  }
 
  return sum / queries.length
}

function ndcgAtK(
  retrieved: string[],
  relevant: string[],
  k: number
): number {
  const topK = retrieved.slice(0, k)
 
  // DCG
  let dcg = 0
  for (let i = 0; i < topK.length; i++) {
    const rel = relevant.includes(topK[i]) ? 1 : 0
    dcg += rel / Math.log2(i + 2)  // i+2 因为 log2(1) = 0
  }
 
  // IDCG（理想排序）
  const idealRelevant = Math.min(relevant.length, k)
  let idcg = 0
  for (let i = 0; i < idealRelevant; i++) {
    idcg += 1 / Math.log2(i + 2)
  }
 
  return idcg > 0 ? dcg / idcg : 0
}

function hitRate(
  queries: Array<{ retrieved: string[]; relevant: string[] }>
): number {
  const hits = queries.filter(({ retrieved, relevant }) =>
    retrieved.some((id) => relevant.includes(id))
  ).length
  return hits / queries.length
}

async function evaluateFaithfulness(
  answer: string,
  contexts: string[]
): Promise<number> {
  // 用 LLM-as-Judge 评估
  const response = await llm.chat({
    messages: [
      {
        role: 'system',
        content: `评估以下回答是否完全基于提供的上下文。
给出 0-1 的评分：
- 1.0：所有信息都能在上下文中找到
- 0.5：大部分信息有依据，但有些细节是编造的
- 0.0：大部分信息是编造的
 
上下文：
${contexts.join('\n---\n')}
 
回答：${answer}
 
只输出数字评分，不要解释。`,
      },
    ],
  })
 
  return parseFloat(response.content)
}

async function evaluateRelevance(
  question: string,
  answer: string
): Promise<number> {
  const response = await llm.chat({
    messages: [
      {
        role: 'system',
        content: `评估以下回答和问题的相关性。
给出 0-1 的评分：
- 1.0：完全回答了问题
- 0.5：部分相关，但没有直接回答
- 0.0：完全无关
 
问题：${question}
回答：${answer}
 
只输出数字评分。`,
      },
    ],
  })
 
  return parseFloat(response.content)
}

async function evaluateCompleteness(
  question: string,
  answer: string,
  contexts: string[]
): Promise<number> {
  const response = await llm.chat({
    messages: [
      {
        role: 'system',
        content: `基于上下文，评估回答的完整性。
给出 0-1 的评分：
- 1.0：完整回答了问题涉及的所有方面
- 0.5：回答了部分方面
- 0.0：遗漏了关键信息
 
问题：${question}
上下文：${contexts.join('\n')}
回答：${answer}
 
只输出数字评分。`,
      },
    ],
  })
 
  return parseFloat(response.content)
}

// 安装: npm install ragas
import { evaluate } from 'ragas'
 
const results = await evaluate({
  question: ['退款多久到账？', '如何修改密码？'],
  answer: [
    '退款一般在 3-5 个工作日到账 [1]。',
    '在设置页面可以修改密码 [2]。',
  ],
  contexts: [
    ['退款一般在 3-5 个工作日到账，退回原支付账户。'],
    ['在「设置」>「账户安全」页面可以修改密码。'],
  ],
  ground_truth: [
    '退款一般在 3-5 个工作日到账。',
    '在设置页面的账户安全选项中可以修改密码。',
  ],
  metrics: ['faithfulness', 'answer_relevancy', 'context_precision', 'context_recall'],
})
 
// results:
// {
//   faithfulness: 0.92,
//   answer_relevancy: 0.88,
//   context_precision: 0.85,
//   context_recall: 0.78,
// }

type EvaluationCase = {
  question: string
  relevantChunkIds: string[]     // 应该检索到的 chunk ID
  groundTruthAnswer?: string     // 标准答案（评估生成质量用）
  category?: string              // 问题分类（按类别分析）
}
 
// 评估数据集示例
const evalDataset: EvaluationCase[] = [
  {
    question: '退款多久到账？',
    relevantChunkIds: ['doc-refund-chunk-2'],
    groundTruthAnswer: '退款一般在 3-5 个工作日到账。',
    category: '退款',
  },
  {
    question: '如何申请退款？',
    relevantChunkIds: ['doc-refund-chunk-0', 'doc-refund-chunk-1'],
    groundTruthAnswer: '在订单页面点击「申请退款」，填写退款原因后提交。',
    category: '退款',
  },
  // ...
]

async function generateEvalDataset(documents: Document[]): Promise<EvaluationCase[]> {
  const cases: EvaluationCase[] = []
 
  for (const doc of documents) {
    // 让 LLM 根据文档内容生成问题和答案
    const response = await llm.chat({
      messages: [
        {
          role: 'system',
          content: `根据以下文档内容，生成 3-5 个用户可能会问的问题，
以及每个问题的答案。格式：
Q: 问题
A: 答案
 
文档内容：
${doc.content}`,
        },
      ],
    })
 
    // 解析 LLM 生成的 QA 对
    const qaPairs = parseQAPairs(response.content)
 
    for (const qa of qaPairs) {
      cases.push({
        question: qa.question,
        relevantChunkIds: [doc.id],  // 简化——实际需要匹配到具体 chunk
        groundTruthAnswer: qa.answer,
      })
    }
  }
 
  return cases
}

// src/services/rag/eval-pipeline.ts
 
export class EvalPipeline {
  async run(config: {
    dataset: EvaluationCase[]
    searchConfig: SearchConfig
    promptTemplate: string
  }): Promise<EvalReport> {
    const { dataset, searchConfig, promptTemplate } = config
    const results: EvalResult[] = []
 
    for (const testCase of dataset) {
      // 1. 检索
      const queryVector = await embed(testCase.question)
      const searchResults = await vectorDB.search(queryVector, searchConfig)
 
      // 2. 检索质量指标
      const retrievedIds = searchResults.map((r) => r.id)
      const recall = recallAtK(retrievedIds, testCase.relevantChunkIds, 5)
 
      // 3. 生成回答
      const context = buildContext(searchResults)
      const answer = await llm.chat({
        messages: [
          { role: 'system', content: promptTemplate.replace('{{context}}', context) },
          { role: 'user', content: testCase.question },
        ],
      })
 
      // 4. 生成质量指标
      const faithfulness = await evaluateFaithfulness(answer, searchResults.map((r) => r.content))
      const relevance = await evaluateRelevance(testCase.question, answer)
 
      results.push({
        question: testCase.question,
        category: testCase.category,
        recall,
        faithfulness,
        relevance,
        retrievedIds,
        answer,
      })
    }
 
    return this.generateReport(results)
  }
 
  private generateReport(results: EvalResult[]): EvalReport {
    const avgRecall = results.reduce((sum, r) => sum + r.recall, 0) / results.length
    const avgFaithfulness = results.reduce((sum, r) => sum + r.faithfulness, 0) / results.length
    const avgRelevance = results.reduce((sum, r) => sum + r.relevance, 0) / results.length
 
    // 按类别分析
    const byCategory = groupBy(results, (r) => r.category ?? 'default')
    const categoryMetrics = Object.entries(byCategory).map(([category, cases]) => ({
      category,
      count: cases.length,
      avgRecall: cases.reduce((sum, r) => sum + r.recall, 0) / cases.length,
      avgFaithfulness: cases.reduce((sum, r) => sum + r.faithfulness, 0) / cases.length,
    }))
 
    return {
      summary: {
        totalCases: results.length,
        avgRecall,
        avgFaithfulness,
        avgRelevance,
      },
      byCategory: categoryMetrics,
      failures: results.filter((r) => r.recall < 0.5 || r.faithfulness < 0.7),
    }
  }
}

# .github/workflows/rag-eval.yml
name: RAG Evaluation
 
on:
  schedule:
    - cron: '0 2 * * 1'  # 每周一凌晨 2 点
  workflow_dispatch:  # 手动触发
 
jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v4
      - run: pnpm install
      - run: pnpm run rag:eval
      - name: Check metrics threshold
        run: |
          # 如果 Recall 低于 0.8，构建失败
          node scripts/check-eval-threshold.js --metric recall --threshold 0.8