21.12-模型质量监控

// src/routes/feedback.ts
app.post('/api/feedback', async (c) => {
  const { requestId, rating, comment } = await c.req.json()
  const userId = c.get('userId')
 
  // 1. 记录反馈
  await c.env.DB.prepare(`
    INSERT INTO feedback (
      request_id, user_id, rating, comment, created_at
    ) VALUES (?, ?, ?, ?, ?)
  `).bind(
    requestId,
    userId,
    rating,  // 1 = 👍, -1 = 👎
    comment,
    Date.now()
  ).run()
 
  // 2. 记录指标
  await recordMetric(c.env, 'feedback.received', {
    rating: String(rating),
    userId,
  }, [1])
 
  return c.json({ success: true })
})

// frontend/src/components/ChatMessage.tsx
function ChatMessage({ message, requestId }) {
  const [feedback, setFeedback] = useState(null)
 
  const handleFeedback = async (rating: 1 | -1) => {
    await fetch('/api/feedback', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ requestId, rating }),
    })
 
    setFeedback(rating)
  }
 
  return (
    <div className="message">
      <div className="content">{message.content}</div>
      {message.role === 'assistant' && (
        <div className="feedback">
          <button
            onClick={() => handleFeedback(1)}
            disabled={feedback !== null}
            className={feedback === 1 ? 'active' : ''}
          >
            👍
          </button>
          <button
            onClick={() => handleFeedback(-1)}
            disabled={feedback !== null}
            className={feedback === -1 ? 'active' : ''}
          >
            👎
          </button>
        </div>
      )}
    </div>
  )
}

// src/routes/admin/quality.ts
app.get('/admin/quality/feedback', async (c) => {
  const days = parseInt(c.req.query('days') || '7')
  const startTime = Date.now() - days * 24 * 60 * 60 * 1000
 
  // 1. 总体满意度
  const overall = await c.env.DB.prepare(`
    SELECT
      COUNT(*) as total,
      SUM(CASE WHEN rating = 1 THEN 1 ELSE 0 END) as positive,
      SUM(CASE WHEN rating = -1 THEN 1 ELSE 0 END) as negative,
      SUM(CASE WHEN rating = 1 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) as satisfaction_rate
    FROM feedback
    WHERE created_at > ?
  `).bind(startTime).first()
 
  // 2. 按模型分组
  const byModel = await c.env.DB.prepare(`
    SELECT
      cr.model,
      COUNT(*) as total,
      SUM(CASE WHEN f.rating = 1 THEN 1 ELSE 0 END) as positive,
      SUM(CASE WHEN f.rating = -1 THEN 1 ELSE 0 END) as negative,
      SUM(CASE WHEN f.rating = 1 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) as satisfaction_rate
    FROM feedback f
    JOIN cost_records cr ON f.request_id = cr.request_id
    WHERE f.created_at > ?
    GROUP BY cr.model
    ORDER BY satisfaction_rate DESC
  `).bind(startTime).all()
 
  // 3. 负面反馈原因分析
  const negativeComments = await c.env.DB.prepare(`
    SELECT comment, COUNT(*) as count
    FROM feedback
    WHERE rating = -1
      AND comment IS NOT NULL
      AND comment != ''
      AND created_at > ?
    GROUP BY comment
    ORDER BY count DESC
    LIMIT 20
  `).bind(startTime).all()
 
  return c.json({
    period: `last ${days} days`,
    overall,
    byModel: byModel.results,
    commonIssues: negativeComments.results,
  })
})

// src/lib/quality-monitor.ts
export async function checkRefusalRate(env: Env) {
  const result = await env.DB.prepare(`
    SELECT
      model,
      COUNT(*) as total_calls,
      SUM(CASE WHEN
        response LIKE '%I cannot%' OR
        response LIKE '%I can't%' OR
        response LIKE '%抱歉，我%' OR
        response LIKE '%我无法%'
      THEN 1 ELSE 0 END) as refusal_count,
      SUM(CASE WHEN
        response LIKE '%I cannot%' OR
        response LIKE '%I can't%' OR
        response LIKE '%抱歉，我%' OR
        response LIKE '%我无法%'
      THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) as refusal_rate
    FROM cost_records
    WHERE created_at > ?
    GROUP BY model
  `).bind(Date.now() - 24 * 60 * 60 * 1000).all()
 
  for (const row of result.results) {
    if (row.refusal_rate > 10) {
      await sendAlert(env, {
        level: 'P2',
        type: 'high_refusal_rate',
        message: `${row.model} 拒绝回答率 ${row.refusal_rate.toFixed(1)}% > 10%`,
      })
    }
  }
 
  return result.results
}

export async function checkRepetitionRate(env: Env) {
  const result = await env.DB.prepare(`
    SELECT
      request_id,
      model,
      response
    FROM cost_records
    WHERE created_at > ?
      AND completion_tokens > 100
  `).bind(Date.now() - 60 * 60 * 1000).all()
 
  const repeated = result.results.filter(row => {
    const response = row.response
    // 检查是否有重复的段落
    const sentences = response.split(/[.!?]/)
    const uniqueSentences = new Set(sentences)
    const repetitionRate = 1 - uniqueSentences.size / sentences.length
 
    return repetitionRate > 0.3  // 30% 以上的句子重复
  })
 
  if (repeated.length > 0) {
    await sendAlert(env, {
      level: 'P2',
      type: 'repetition_detected',
      message: `检测到 ${repeated.length} 个请求存在重复输出`,
    })
  }
 
  return repeated
}

export async function checkHallucinationRate(env: Env) {
  // 采样 10% 的负面反馈，用另一个 LLM 检测是否是幻觉
  const samples = await env.DB.prepare(`
    SELECT request_id, model, messages, response
    FROM cost_records
    WHERE request_id IN (
      SELECT request_id FROM feedback WHERE rating = -1
    )
    AND created_at > ?
    ORDER BY RANDOM()
    LIMIT 100
  `).bind(Date.now() - 7 * 24 * 60 * 60 * 1000).all()
 
  let hallucinationCount = 0
 
  for (const sample of samples.results) {
    // 用另一个 LLM 检测是否是幻觉
    const detection = await detectHallucination(
      sample.messages,
      sample.response,
      env
    )
 
    if (detection.isHallucination) {
      hallucinationCount++
    }
  }
 
  const hallucinationRate = hallucinationCount / samples.results.length * 100
 
  if (hallucinationRate > 20) {
    await sendAlert(env, {
      level: 'P1',
      type: 'high_hallucination_rate',
      message: `幻觉率 ${hallucinationRate.toFixed(1)}% > 20%`,
    })
  }
 
  return { hallucinationRate, sampleCount: samples.results.length }
}
 
async function detectHallucination(
  messages: string,
  response: string,
  env: Env
): Promise<{ isHallucination: boolean; reason: string }> {
  const result = await callLLM({
    model: 'gpt-4-turbo',
    messages: [
      {
        role: 'system',
        content: '你是一个事实核查专家。请判断以下回答是否包含幻觉（编造不存在的事实）。只回答 JSON：{"isHallucination": true/false, "reason": "..."}',
      },
      {
        role: 'user',
        content: `问题：${messages}\n\n回答：${response}`,
      },
    ],
  }, env)
 
  return JSON.parse(result.choices[0].message.content)
}

export async function analyzeResponseLength(env: Env) {
  const result = await env.DB.prepare(`
    SELECT
      model,
      AVG(completion_tokens) as avg_tokens,
      MIN(completion_tokens) as min_tokens,
      MAX(completion_tokens) as max_tokens,
      APPROX_QUANTILE(0.50, completion_tokens) as p50_tokens,
      APPROX_QUANTILE(0.90, completion_tokens) as p90_tokens
    FROM cost_records
    WHERE created_at > ?
    GROUP BY model
  `).bind(Date.now() - 24 * 60 * 60 * 1000).all()
 
  // 检查异常
  for (const row of result.results) {
    if (row.avg_tokens < 50) {
      await sendAlert(env, {
        level: 'P3',
        type: 'short_responses',
        message: `${row.model} 平均响应长度 ${row.avg_tokens} tokens，可能过于简短`,
      })
    }
  }
 
  return result.results
}

export async function analyzeFinishReasons(env: Env) {
  const result = await env.DB.prepare(`
    SELECT
      model,
      finish_reason,
      COUNT(*) as count,
      COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (PARTITION BY model) as percentage
    FROM cost_records
    WHERE created_at > ?
    GROUP BY model, finish_reason
    ORDER BY model, count DESC
  `).bind(Date.now() - 24 * 60 * 60 * 1000).all()
 
  // 检查 length（截断）比例
  const lengthRatio = result.results
    .filter(r => r.finish_reason === 'length')
    .reduce((sum, r) => sum + r.percentage, 0) / result.results.length
 
  if (lengthRatio > 20) {
    await sendAlert(env, {
      level: 'P2',
      type: 'high_truncation_rate',
      message: `${lengthRatio.toFixed(1)}% 的响应被截断（finish_reason=length）`,
    })
  }
 
  return result.results
}

// src/cron/quality-review.ts
export default {
  async scheduled(event, env, ctx) {
    // 每周一 00:00 执行
 
    // 1. 随机采样 100 个请求
    const samples = await env.DB.prepare(`
      SELECT request_id, model, messages, response, user_id
      FROM cost_records
      WHERE created_at > ?
      ORDER BY RANDOM()
      LIMIT 100
    `).bind(Date.now() - 7 * 24 * 60 * 60 * 1000).all()
 
    // 2. 创建评审任务
    for (const sample of samples.results) {
      await env.DB.prepare(`
        INSERT INTO quality_reviews (
          request_id, model, messages, response, user_id,
          status, created_at
        ) VALUES (?, ?, ?, ?, ?, 'pending', ?)
      `).bind(
        sample.request_id,
        sample.model,
        sample.messages,
        sample.response,
        sample.user_id,
        Date.now()
      ).run()
    }
 
    // 3. 通知评审人员
    await sendSlackAlert(env, {
      level: 'INFO',
      type: 'quality_review_ready',
      message: `📋 本周有 ${samples.results.length} 个请求待评审`,
    })
  },
}

// src/routes/admin/quality-review.ts
app.post('/admin/quality-review/:id', async (c) => {
  const id = c.req.param('id')
  const { accuracy, relevance, completeness, safety, comment } = await c.req.json()
  const reviewerId = c.get('userId')
 
  await c.env.DB.prepare(`
    UPDATE quality_reviews
    SET
      accuracy = ?,
      relevance = ?,
      completeness = ?,
      safety = ?,
      comment = ?,
      reviewer_id = ?,
      status = 'completed',
      reviewed_at = ?
    WHERE id = ?
  `).bind(
    accuracy,      // 1-5 分
    relevance,     // 1-5 分
    completeness,  // 1-5 分
    safety,        // 1-5 分
    comment,
    reviewerId,
    Date.now(),
    id
  ).run()
 
  return c.json({ success: true })
})

// src/routes/admin/quality-trends.ts
app.get('/admin/quality/trends', async (c) => {
  const days = parseInt(c.req.query('days') || '30')
  const startTime = Date.now() - days * 24 * 60 * 60 * 1000
 
  // 1. 满意度趋势
  const satisfactionTrend = await c.env.DB.prepare(`
    SELECT
      date(created_at / 1000, 'unixepoch') as date,
      COUNT(*) as total,
      SUM(CASE WHEN rating = 1 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) as satisfaction_rate
    FROM feedback
    WHERE created_at > ?
    GROUP BY date
    ORDER BY date
  `).bind(startTime).all()
 
  // 2. 各模型满意度对比
  const modelComparison = await c.env.DB.prepare(`
    SELECT
      cr.model,
      COUNT(*) as total,
      SUM(CASE WHEN f.rating = 1 THEN 1 ELSE 0 END) * 100.0 / NULLIF(COUNT(*), 0) as satisfaction_rate,
      AVG(f.rating) as avg_rating
    FROM feedback f
    JOIN cost_records cr ON f.request_id = cr.request_id
    WHERE f.created_at > ?
    GROUP BY cr.model
    ORDER BY satisfaction_rate DESC
  `).bind(startTime).all()
 
  return c.json({
    period: `last ${days} days`,
    satisfactionTrend: satisfactionTrend.results,
    modelComparison: modelComparison.results,
  })
})

// src/cron/quality-monitor.ts
export default {
  async scheduled(event, env, ctx) {
    // 每 6 小时执行一次
 
    const metrics = {
      refusalRate: await checkRefusalRate(env),
      repetitionRate: await checkRepetitionRate(env),
      responseLength: await analyzeResponseLength(env),
      finishReasons: await analyzeFinishReasons(env),
    }
 
    // 记录到指标系统
    for (const [metric, values] of Object.entries(metrics)) {
      for (const value of values) {
        await recordMetric(env, `quality.${metric}`, {
          model: value.model,
        }, [value.value])
      }
    }
 
    // 生成报告
    const report = generateQualityReport(metrics)
    await sendSlackAlert(env, {
      level: 'INFO',
      type: 'quality_report',
      message: report,
    })
  },
}
 
function generateQualityReport(metrics: any): string {
  return `📊 模型质量报告\n\n` +
    `拒绝回答率: ${metrics.refusalRate.map(r => `${r.model}: ${r.refusal_rate.toFixed(1)}%`).join(', ')}\n` +
    `平均响应长度: ${metrics.responseLength.map(r => `${r.model}: ${r.avg_tokens.toFixed(0)} tokens`).join(', ')}\n` +
    `截断率: ${metrics.finishReasons.filter(r => r.finish_reason === 'length').map(r => `${r.model}: ${r.percentage.toFixed(1)}%`).join(', ')}`
}