内容社区审核案例：规则、模型和人工如何协作

// ❌ 坏做法：直接调模型，不传社区上下文
async function moderate(content: string): Promise<ModerationResult> {
  const result = await moderationModel.classify(content)
  if (result.score > 0.7) {
    return { action: 'delete', reason: result.category }
  }
  return { action: 'pass' }
}
 
// ✅ 好做法：传入社区属性，用置信度区间决定处理路径
async function moderate(
  content: string,
  context: CommunityContext
): Promise<ModerationResult> {
  // 规则层先过滤——命中黑名单直接拦截
  const ruleHit = await ruleEngine.match(content, context.communityId)
  if (ruleHit) {
    return { action: 'block', reason: ruleHit.ruleId, source: 'rule' }
  }
 
  // 模型层带上下文分类
  const result = await moderationModel.classify(content, {
    communityType: context.communityType, // 'gaming' | 'parenting' | 'general'
    contentTags: context.tags,
    userHistory: context.userRiskLevel,
  })
 
  // 高置信度自动执行
  if (result.confidence > 0.9 && result.severity === 'critical') {
    return { action: 'block', reason: result.category, source: 'model' }
  }
 
  // 中等置信度进人工队列
  if (result.confidence > 0.6) {
    return { action: 'review', reason: result.category, source: 'model' }
  }
 
  return { action: 'pass', source: 'model' }
}

// ❌ 坏做法：关键词命中直接删除
function keywordFilter(text: string): boolean {
  const blockedWords = ['hit', 'kill', 'hurt', 'violence']
  return blockedWords.some(word =>
    text.toLowerCase().includes(word)
  )
  // "my child hit me" → true → 删除
}
 
// ✅ 好做法：关键词命中后交给模型做语义判断
async function semanticFilter(
  text: string,
  communityId: string
): Promise<FilterResult> {
  const keywordHits = keywordMatcher.scan(text)
 
  // 没有命中关键词，走正常流程
  if (keywordHits.length === 0) {
    return { action: 'pass' }
  }
 
  // 命中关键词，交给模型判断语境
  const semanticResult = await semanticModel.analyze(text, {
    communityId,
    // 告诉模型：这些词命中了规则，但请判断实际语境
    triggeredKeywords: keywordHits,
    contextCategories: ['violence_description', 'personal_experience',
                        'educational', 'news_reporting', 'fiction'],
  })
 
  // 模型判断为「真实暴力威胁」才拦截
  if (semanticResult.category === 'violence_description'
      && semanticResult.confidence > 0.85) {
    return { action: 'block', reason: 'real_violence_threat' }
  }
 
  // 其他语境放行，但记录日志供审计
  return {
    action: 'pass',
    note: `关键词命中但语境为「${semanticResult.category}」，放行`,
    auditTrail: { keywordHits, semanticResult },
  }
}

// ❌ 坏做法：只存最终结果，不记录决策过程
async function legacyModerate(content: string): Promise<void> {
  const result = await model.classify(content)
  if (result.violation) {
    await db.contents.update(contentId, { status: 'deleted' })
    // 只存了 status，没有 result.score、result.reason、model version
  }
}
 
// ✅ 好做法：完整审计日志，支持回溯和申诉
interface AuditLog {
  contentId: string
  contentHash: string        // 内容版本校验
  action: 'pass' | 'block' | 'review'
  ruleHits: Array<{
    ruleId: string
    ruleVersion: string
    matchedText: string
  }>
  modelDecision: {
    modelId: string
    modelVersion: string
    promptVersion: string    // prompt 版本也要记
    category: string
    confidence: number
    reasoning: string        // ReAct 推理链
  }
  humanReview?: {
    reviewerId: string
    decision: 'uphold' | 'override'
    reason: string
    reviewedAt: string
  }
  appeal?: {
    submittedAt: string
    reason: string
    resolvedAt?: string
    resolution?: 'uphold' | 'revert'
  }
  timestamp: string
}
 
async function moderateWithAudit(
  content: string,
  context: ModerationContext
): Promise<AuditLog> {
  const ruleHits = await ruleEngine.match(content, context)
  const modelResult = await model.classify(content, context)
 
  const log: AuditLog = {
    contentId: context.contentId,
    contentHash: hash(content),
    action: determineAction(ruleHits, modelResult),
    ruleHits: ruleHits.map(r => ({
      ruleId: r.id,
      ruleVersion: r.version,
      matchedText: r.matchedText,
    })),
    modelDecision: {
      modelId: modelResult.modelId,
      modelVersion: modelResult.modelVersion,
      promptVersion: context.promptVersion,
      category: modelResult.category,
      confidence: modelResult.confidence,
      reasoning: modelResult.reasoning,
    },
    timestamp: new Date().toISOString(),
  }
 
  // 审计日志和内容决策一起写入，保证一致性
  await auditStore.append(log)
  await executeAction(log.action, context)
 
  return log
}

// ❌ 坏做法：所有内容都调 LLM，成本爆炸
async function moderateAllWithLLM(contents: Content[]): Promise<void> {
  for (const content of contents) {
    // 每条内容都调一次 LLM，包括明显的垃圾广告
    const result = await llm.classify(content.text, MODERATION_PROMPT)
    await execute(result)
  }
  // 问题：日均 100 万条内容，成本 = 100 万 × ¥0.05 = ¥50,000/天
}
 
// ✅ 好做法：分级处理，LLM 只处理需要语义理解的中等风险内容
async function moderateTiered(contents: Content[]): Promise<void> {
  for (const content of contents) {
    // 第一层：规则引擎处理高确定性的内容（约 60%）
    const ruleResult = await ruleEngine.evaluate(content)
    if (ruleResult.decided) {
      await execute(ruleResult)
      continue
    }
 
    // 第二层：轻量分类模型处理明显的违规/合规（约 25%）
    const quickResult = await quickClassifier.predict(content.features)
    if (quickResult.confidence > 0.95) {
      await execute(quickResult)
      continue
    }
 
    // 第三层：LLM 只处理剩余 15% 的灰色地带
    const llmResult = await llm.classify(content.text, {
      prompt: MODERATION_PROMPT,
      context: { communityType: content.communityType },
      temperature: 0,  // 生产环境必须锁定
    })
    await execute(llmResult)
  }
  // 日均 100 万条：规则 60 万免费 + 分类 25 万 × ¥0.001 + LLM 15 万 × ¥0.05
  // 成本 = ¥250 + ¥7,500 = ¥7,750/天，降 85%
}

// ❌ 坏做法：只监控整体准确率
function reportAccuracy(stats: ModerationStats): string {
  const accuracy = (stats.correctDecisions / stats.totalDecisions) * 100
  return `审核准确率：${accuracy.toFixed(2)}%`
  // "准确率 95%" 掩盖了 5% 误杀的严重性
}
 
// ✅ 好做法：多维度指标看板
interface ModerationMetrics {
  // 准确性指标
  falsePositiveRate: number   // 误杀率
  falseNegativeRate: number   // 漏放率
  precision: number           // 精确率：拦截内容中真正违规的比例
  recall: number              // 召回率：真正违规内容中被拦截的比例
 
  // 效率指标
  autoResolutionRate: number  // 自动处理率（不需要人工的比例）
  avgReviewTime: number       // 平均人工复核时长（秒）
  queueDepth: number          // 待复核队列深度
 
  // 用户体验指标
  appealRate: number          // 申诉率
  appealReversalRate: number  // 申诉撤销率
  avgAppealResolution: number // 平均申诉处理时长（小时）
 
  // 社区健康指标
  repeatViolationRate: number // 重复违规率
  retainedUsersAfterFalsePositive: number // 误杀后 7 日留存
}
 
function calculateMetrics(
  decisions: AuditLog[],
  appeals: AppealRecord[],
  groundTruth: GroundTruthLabel[]
): ModerationMetrics {
  const tp = countTruePositives(decisions, groundTruth)
  const fp = countFalsePositives(decisions, groundTruth)
  const fn = countFalseNegatives(decisions, groundTruth)
  const tn = countTrueNegatives(decisions, groundTruth)
 
  return {
    falsePositiveRate: fp / (fp + tn),
    falseNegativeRate: fn / (fn + tp),
    precision: tp / (tp + fp),
    recall: tp / (tp + fn),
    autoResolutionRate: countAutoResolved(decisions) / decisions.length,
    avgReviewTime: averageHumanReviewTime(decisions),
    queueDepth: currentQueueDepth(),
    appealRate: appeals.length / decisions.length,
    appealReversalRate: countReversed(appeals) / appeals.length,
    avgAppealResolution: averageAppealResolutionTime(appeals),
    repeatViolationRate: calculateRepeatRate(decisions),
    retainedUsersAfterFalsePositive: measureRetentionAfterFP(decisions),
  }
}