21.13-生产故障排查流程

// 1. 检查健康状态
const health = await fetch('https://your-worker.com/health')
const status = await health.json()
 
if (status.status === 'error') {
  console.error('服务不可用:', status.checks)
}
 
// 2. 检查错误率
const errorRate = await env.ANALYTICS.sql(`
  SELECT
    SUM(CASE WHEN blob3 >= '500' THEN _sample_interval ELSE 0 END) * 100.0 /
    SUM(_sample_interval) AS error_rate
  FROM ai_gateway_metrics
  WHERE blob1 = 'http.request'
    AND timestamp > NOW() - INTERVAL '5' MINUTE
`)
 
console.log('当前错误率:', errorRate[0].error_rate)
 
// 3. 检查 P99 延迟
const p99 = await env.ANALYTICS.sql(`
  SELECT APPROX_QUANTILE(0.99, double1) AS p99_ms
  FROM ai_gateway_metrics
  WHERE blob1 = 'llm.call.success'
    AND timestamp > NOW() - INTERVAL '5' MINUTE
`)
 
console.log('P99 延迟:', p99[0].p99_ms, 'ms')

// 1. 从告警提取关键信息
const alertInfo = {
  type: 'high_error_rate',
  errorRate: '15%',
  startTime: '2026-06-21T10:30:00Z',
}
 
// 2. 查询对应时间段的错误日志
const errorLogs = await env.LOGS.query(`
  SELECT
    timestamp,
    level,
    event,
    requestId,
    error.message,
    error.stack
  FROM logs
  WHERE level = 'error'
    AND timestamp > '${alertInfo.startTime}'
  ORDER BY timestamp DESC
  LIMIT 100
`)
 
// 3. 分析错误模式
const errorPattern = analyzeErrorPattern(errorLogs)
console.log('错误模式:', errorPattern)

// 1. 找到一个错误的 requestId
const errorRequestId = errorLogs.results[0].requestId
 
// 2. 查询该请求的完整 trace
const trace = await env.TRACES.query(`
  SELECT
    traceId,
    spanId,
    parentSpanId,
    operationName,
    startTime,
    duration,
    tags,
    logs
  FROM traces
  WHERE requestId = '${errorRequestId}'
  ORDER BY startTime
`)
 
// 3. 可视化 trace 链路
console.log('Trace 链路:')
for (const span of trace.results) {
  const indent = '  '.repeat(span.depth)
  console.log(`${indent}${span.operationName} (${span.duration}ms)`)
}

// 1. 从 trace 找到最慢的 span
const slowSpan = trace.results.reduce((slowest, span) =>
  span.duration > slowest.duration ? span : slowest
)
 
console.log('最慢的操作:', slowSpan.operationName)
console.log('耗时:', slowSpan.duration, 'ms')
 
// 2. 查看该 span 的日志
const spanLogs = await env.LOGS.query(`
  SELECT *
  FROM logs
  WHERE requestId = '${errorRequestId}'
    AND timestamp BETWEEN ${slowSpan.startTime} AND ${slowSpan.startTime + slowSpan.duration}
`)
 
console.log('相关日志:', spanLogs.results)
 
// 3. 查看代码（需要知道 operationName 对应的代码位置）
// 例如：operationName = 'llm.call.openai' 对应 src/lib/llm.ts:callOpenAI()

SELECT
  model,
  AVG(duration) as avg_ms,
  APPROX_QUANTILE(0.99, duration) as p99_ms,
  COUNT(*) as call_count
FROM ai_gateway_metrics
WHERE blob1 = 'llm.call.success'
  AND timestamp > NOW() - INTERVAL '1' HOUR
GROUP BY model

const byModel = await env.ANALYTICS.sql(`
  SELECT
    model,
    AVG(duration) as avg_ms,
    COUNT(*) as call_count
  FROM ai_gateway_metrics
  WHERE blob1 = 'llm.call.success'
    AND timestamp > NOW() - INTERVAL '1' HOUR
  GROUP BY model
  HAVING avg_ms > 10000
`)
 
if (byModel.results.length > 0) {
  console.log('延迟异常的模型:', byModel.results)
}

// src/lib/llm-fallback.ts
export async function callLLMWithFallback(
  body: ChatRequest,
  env: Env
): Promise<ChatResponse> {
  try {
    // 尝试主模型
    return await callLLM(body, env)
  } catch (err) {
    if (err.status === 504 || err.status === 503) {
      // 超时或服务不可用，降级到备用模型
      console.warn('主模型不可用，降级到备用模型')
 
      return await callLLM({
        ...body,
        model: 'gpt-3.5-turbo',  // 降级到便宜快速的模型
      }, env)
    }
 
    throw err
  }
}

curl https://your-worker.com/health

const dbErrors = await env.LOGS.query(`
  SELECT error.message, error.stack
  FROM logs
  WHERE event = 'database.query'
    AND level = 'error'
    AND timestamp > NOW() - INTERVAL '5' MINUTE
  LIMIT 10
`)
 
console.log('数据库错误:', dbErrors.results)

// src/lib/db-fallback.ts
export async function queryWithCache<T>(
  env: Env,
  query: string,
  params: any[],
  cacheKey: string,
  ttl: number = 300
): Promise<T> {
  try {
    // 尝试查询数据库
    return await env.DB.prepare(query).bind(...params).first()
  } catch (err) {
    console.error('数据库查询失败:', err.message)
 
    // 降级到缓存
    const cached = await env.CACHE.get(cacheKey)
    if (cached) {
      console.warn('降级到缓存数据')
      return JSON.parse(cached)
    }
 
    throw err
  }
}

SELECT
  user_id,
  SUM(total_tokens) as total_tokens,
  SUM(estimated_cost) as total_cost,
  COUNT(*) as call_count
FROM cost_records
WHERE created_at > NOW() - INTERVAL '1' HOUR
GROUP BY user_id
ORDER BY total_tokens DESC
LIMIT 10

SELECT
  model,
  SUM(total_tokens) as total_tokens,
  COUNT(*) as call_count
FROM cost_records
WHERE created_at > NOW() - INTERVAL '1' HOUR
GROUP BY model
ORDER BY total_tokens DESC

const suspiciousRequests = await env.DB.prepare(`
  SELECT user_id, messages, total_tokens
  FROM cost_records
  WHERE total_tokens > 100000
    AND created_at > ?
`).bind(Date.now() - 60 * 60 * 1000).all()
 
for (const req of suspiciousRequests.results) {
  console.log('可疑请求:', req.user_id, req.messages)
}

await env.DB.prepare(`
  UPDATE users SET status = 'disabled' WHERE id = ?
`).bind(suspiciousUserId).run()

SELECT
  time_slice(timestamp, 5, 'minute') AS time_bucket,
  SUM(CASE WHEN blob1 = 'cache.hit' THEN _sample_interval ELSE 0 END) AS hits,
  SUM(CASE WHEN blob1 = 'cache.miss' THEN _sample_interval ELSE 0 END) AS misses,
  hits * 100.0 / (hits + misses) AS hit_rate
FROM ai_gateway_metrics
WHERE blob1 IN ('cache.hit', 'cache.miss')
  AND timestamp > NOW() - INTERVAL '30' MINUTE
GROUP BY time_bucket
ORDER BY time_bucket

// 检查缓存过期时间分布
const cacheKeys = await env.CACHE.list({ prefix: 'chat:' })
 
const expirationDistribution = cacheKeys.keys.reduce((dist, key) => {
  const expiration = key.expiration
  const bucket = Math.floor(expiration / 3600) * 3600  // 按小时分桶
  dist[bucket] = (dist[bucket] || 0) + 1
  return dist
}, {})
 
console.log('缓存过期时间分布:', expirationDistribution)

// src/lib/cache-warmup.ts
export async function warmupCache(env: Env) {
  // 查询最近 1 小时的热门请求
  const hotRequests = await env.DB.prepare(`
    SELECT messages, model, COUNT(*) as count
    FROM cost_records
    WHERE created_at > ?
    GROUP BY messages, model
    ORDER BY count DESC
    LIMIT 100
  `).bind(Date.now() - 60 * 60 * 1000).all()
 
  // 预热缓存
  for (const req of hotRequests.results) {
    const cacheKey = generateCacheKey(req)
    const cached = await env.CACHE.get(cacheKey)
 
    if (!cached) {
      // 缓存不存在，重新生成
      const result = await callLLM({
        model: req.model,
        messages: JSON.parse(req.messages),
      }, env)
 
      await env.CACHE.put(cacheKey, JSON.stringify(result), {
        expirationTtl: 3600 + Math.random() * 600,  // 随机化过期时间
      })
    }
  }
}

# 查看最近的部署
npx wrangler deployments list
 
# 回滚到上一个版本
npx wrangler rollback

// src/routes/chat.ts
app.post('/v1/chat/completions', async (c) => {
  // 检查是否启用降级模式
  const degradedMode = await c.env.FEATURE_FLAGS.get('degraded_mode')
 
  if (degradedMode === 'true') {
    // 降级模式：跳过缓存、简化逻辑
    const result = await callLLM(body, c.env)
    return c.json(result)
  }
 
  // 正常模式
  // ...
})

// src/middleware/rate-limit.ts
export async function rateLimit(c: Context, next: Next) {
  const userId = c.get('userId')
  const key = `ratelimit:${userId}`
 
  const current = await c.env.RATE_LIMIT.get(key, 'json') || {
    count: 0,
    resetAt: Date.now() + 60 * 1000,
  }
 
  if (Date.now() > current.resetAt) {
    current.count = 0
    current.resetAt = Date.now() + 60 * 1000
  }
 
  // 动态调整限流阈值
  const errorRate = await getErrorRate(c.env)
  const limit = errorRate > 10 ? 10 : 60  // 错误率高时降低限流
 
  if (current.count >= limit) {
    return c.json({ error: 'Rate limit exceeded' }, 429)
  }
 
  current.count++
  await c.env.RATE_LIMIT.put(key, JSON.stringify(current))
 
  await next()
}

# 故障复盘报告
 
## 基本信息
 
- 故障时间：2026-06-21 10:30 - 11:00 (30 分钟)
- 影响范围：所有用户，约 1000 个请求失败
- 故障级别：P1
 
## 故障描述
 
10:30 收到告警，HTTP 错误率升高到 15%。用户报告 AI 对话功能无法使用。
 
## 时间线
 
- 10:30 收到错误率告警
- 10:35 确认为 LLM API 超时
- 10:40 检查 OpenAI 状态页面，确认 OpenAI 故障
- 10:45 启用降级策略，切换到备用模型
- 10:50 错误率恢复正常
- 11:00 OpenAI 恢复，切换回主模型
 
## 根因分析
 
OpenAI API 在 10:30 - 10:50 期间出现服务故障，导致所有调用超时。
 
## 改进措施
 
1. **短期**：添加 OpenAI 状态监控，在 OpenAI 故障时自动降级
2. **中期**：实现多模型自动切换，不依赖单一提供商
3. **长期**：建立模型抽象层，支持快速切换不同模型
 
## 负责人
 
- 故障处理：张三
- 复盘主持：李四
- 改进跟踪：王五

// src/lib/incident-toolkit.ts
 
// 1. 快速诊断
export async function quickDiagnosis(env: Env) {
  const [health, errorRate, p99, activeUsers] = await Promise.all([
    fetch('https://your-worker.com/health').then(r => r.json()),
    getErrorRate(env),
    getP99Latency(env),
    getActiveUsers(env),
  ])
 
  return {
    health,
    errorRate,
    p99,
    activeUsers,
    timestamp: new Date().toISOString(),
  }
}
 
// 2. 错误分析
export async function analyzeErrors(env: Env, minutes: number = 30) {
  const errors = await env.LOGS.query(`
    SELECT
      event,
      error.message,
      COUNT(*) as count
    FROM logs
    WHERE level = 'error'
      AND timestamp > NOW() - INTERVAL '${minutes}' MINUTE
    GROUP BY event, error.message
    ORDER BY count DESC
    LIMIT 20
  `)
 
  return errors.results
}
 
// 3. 慢请求分析
export async function analyzeSlowRequests(env: Env, thresholdMs: number = 5000) {
  const slowRequests = await env.ANALYTICS.sql(`
    SELECT
      method,
      path,
      userId,
      requestId,
      duration
    FROM ai_gateway_metrics
    WHERE blob1 = 'http.request'
      AND double1 > ${thresholdMs}
      AND timestamp > NOW() - INTERVAL '30' MINUTE
    ORDER BY duration DESC
    LIMIT 50
  `)
 
  return slowRequests.results
}
 
// 4. 成本异常分析
export async function analyzeCostAnomaly(env: Env) {
  const today = await env.DB.prepare(`
    SELECT SUM(estimated_cost) as total_cost
    FROM cost_records
    WHERE created_at > date('now')
  `).first()
 
  const yesterday = await env.DB.prepare(`
    SELECT SUM(estimated_cost) as total_cost
    FROM cost_records
    WHERE created_at BETWEEN date('now', '-1 day') AND date('now')
  `).first()
 
  const increase = (today.total_cost - yesterday.total_cost) / yesterday.total_cost * 100
 
  return {
    today: today.total_cost,
    yesterday: yesterday.total_cost,
    increase: `${increase.toFixed(1)}%`,
  }
}
 
// 5. 生成故障报告
export async function generateIncidentReport(env: Env): Promise<string> {
  const diagnosis = await quickDiagnosis(env)
  const errors = await analyzeErrors(env)
  const slowRequests = await analyzeSlowRequests(env)
  const costAnomaly = await analyzeCostAnomaly(env)
 
  return `
# 故障诊断报告
 
生成时间: ${diagnosis.timestamp}
 
## 系统状态
 
- 健康状态: ${diagnosis.health.status}
- 错误率: ${diagnosis.errorRate.toFixed(2)}%
- P99 延迟: ${diagnosis.p99.toFixed(0)}ms
- 活跃用户: ${diagnosis.activeUsers}
 
## 错误分布
 
${errors.map(e => `- ${e.event}: ${e.count} 次 (${e.message})`).join('\n')}
 
## 慢请求 Top 10
 
${slowRequests.slice(0, 10).map(r => `- ${r.method} ${r.path}: ${r.duration}ms`).join('\n')}
 
## 成本异常
 
今日成本: $${costAnomaly.today.toFixed(2)}
昨日成本: $${costAnomaly.yesterday.toFixed(2)}
增长率: ${costAnomaly.increase}
  `.trim()
}