# ===== 499 相關(guān)配置 =====
# 讓 Nginx 記錄 499 狀態(tài)碼（默認(rèn)不記錄）
# 在 http/server/location 中配置
access_log /var/log/nginx/access.log combined;
# 或者配置 proxy_ignore_client_abort
# on: 客戶端斷開(kāi)時(shí)繼續(xù)等待上游響應(yīng)（可能浪費(fèi)資源）
# off: 客戶端斷開(kāi)時(shí)立即關(guān)閉上游連接（默認(rèn)）
proxy_ignore_client_abort off;
# 查看 499 錯(cuò)誤日志
grep " 499 " /var/log/nginx/access.log | tail -100
# 統(tǒng)計(jì) 499 錯(cuò)誤數(shù)量
grep " 499 " /var/log/nginx/access.log | wc -l
# 統(tǒng)計(jì) 499 錯(cuò)誤的接口分布
grep " 499 " /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn | head -20

# ===== 502 錯(cuò)誤日志示例 =====
# 錯(cuò)誤日志示例
# 2026/06/04 10:00:00 [error] 12345#0: *67890 connect() failed (111: Connection refused) while connecting to upstream, client: 192.168.1.100, server: api.example.com, request: "GET /api/users HTTP/1.1", upstream: "http://127.0.0.1:8000/api/users", host: "api.example.com"
# 常見(jiàn)錯(cuò)誤原因
# 1. Connection refused - 服務(wù)未啟動(dòng)或端口不對(duì)
# 2. No such file or directory - Unix socket 不存在
# 3. Permission denied - 權(quán)限問(wèn)題
# 4. Connection reset by peer - 服務(wù)崩潰
# 查看 502 錯(cuò)誤日志
grep " 502 " /var/log/nginx/access.log | tail -100
grep "connect() failed" /var/log/nginx/error.log | tail -100
# 統(tǒng)計(jì) 502 錯(cuò)誤的上游地址
grep " 502 " /var/log/nginx/access.log | grep -oP 'upstream: "\K[^"]+' | sort | uniq -c
# 檢查上游服務(wù)狀態(tài)
systemctl status your-service
ps aux | grep your-service
netstat -tlnp | grep 8000
ss -tlnp | grep 8000

# ===== 504 錯(cuò)誤日志示例 =====
# 錯(cuò)誤日志示例
# 2026/06/04 10:00:00 [error] 12345#0: *67890 upstream timed out (110: Connection timed out) while reading response header from upstream, client: 192.168.1.100, server: api.example.com, request: "GET /api/slow-query HTTP/1.1", upstream: "http://127.0.0.1:8000/api/slow-query", host: "api.example.com"
# 查看 504 錯(cuò)誤日志
grep " 504 " /var/log/nginx/access.log | tail -100
grep "upstream timed out" /var/log/nginx/error.log | tail -100
# 統(tǒng)計(jì) 504 錯(cuò)誤的接口
grep " 504 " /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn | head -20
# 統(tǒng)計(jì) 504 錯(cuò)誤的時(shí)間分布（找出高峰時(shí)段）
grep " 504 " /var/log/nginx/access.log | awk '{print $4}' | cut -d: -f1-2 | sort | uniq -c

# ===== Nginx 超時(shí)配置詳解 =====
http {
    # 客戶端超時(shí)
    client_body_timeout 60s;        # 讀取請(qǐng)求體超時(shí)
    client_header_timeout 60s;      # 讀取請(qǐng)求頭超時(shí)
    send_timeout 60s;               # 響應(yīng)發(fā)送超時(shí)
    # keepalive 超時(shí)
    keepalive_timeout 65s;          # 長(zhǎng)連接超時(shí)
    # 上游超時(shí)（關(guān)鍵配置）
    proxy_connect_timeout 60s;      # 與上游建立連接超時(shí)
    proxy_send_timeout 60s;         # 向上游發(fā)送請(qǐng)求超時(shí)
    proxy_read_timeout 60s;         # 等待上游響應(yīng)超時(shí)（最常見(jiàn)）
    # FastCGI 超時(shí)（PHP 等）
    fastcgi_connect_timeout 60s;
    fastcgi_send_timeout 60s;
    fastcgi_read_timeout 60s;
    # uwsgi 超時(shí)（Python 等）
    uwsgi_connect_timeout 60s;
    uwsgi_send_timeout 60s;
    uwsgi_read_timeout 60s;
    upstream backend {
        server 127.0.0.1:8000;
        server 127.0.0.1:8001 backup;
        # 保持連接池
        keepalive 32;
        keepalive_timeout 60s;
        keepalive_requests 1000;
    }
    server {
        listen 80;
        server_name api.example.com;
        # 針對(duì)特定接口調(diào)整超時(shí)
        location /api/ {
            proxy_pass http://backend;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            # 繼承全局超時(shí)或單獨(dú)設(shè)置
            proxy_connect_timeout 60s;
            proxy_send_timeout 60s;
            proxy_read_timeout 60s;
        }
        # 慢接口單獨(dú)配置更長(zhǎng)超時(shí)
        location /api/slow-query {
            proxy_pass http://backend;
            proxy_read_timeout 300s;  # 5 分鐘
        }
        # 大文件上傳
        location /api/upload {
            proxy_pass http://backend;
            client_max_body_size 100m;
            client_body_timeout 300s;
            proxy_read_timeout 300s;
        }
    }
}

# ===== Nginx 錯(cuò)誤處理配置 =====
server {
    listen 80;
    server_name api.example.com;
    # 自定義錯(cuò)誤頁(yè)面
    error_page 499 = @handle_499;
    error_page 502 = @handle_502;
    error_page 504 = @handle_504;
    location @handle_499 {
        default_type application/json;
        return 499 '{"code": 499, "message": "客戶端取消請(qǐng)求"}';
    }
    location @handle_502 {
        default_type application/json;
        return 502 '{"code": 502, "message": "服務(wù)暫時(shí)不可用"}';
    }
    location @handle_504 {
        default_type application/json;
        return 504 '{"code": 504, "message": "請(qǐng)求超時(shí)，請(qǐng)稍后重試"}';
    }
    # 重試機(jī)制
    location /api/ {
        proxy_pass http://backend;
        # 發(fā)生錯(cuò)誤時(shí)重試其他服務(wù)器
        proxy_next_upstream error timeout http_502 http_503 http_504;
        proxy_next_upstream_tries 3;      # 最多重試 3 次
        proxy_next_upstream_timeout 30s;  # 重試總超時(shí) 30 秒
        # 連接超時(shí)不重試（防止雪崩）
        # proxy_next_upstream_timeout 0;
    }
    # 健康檢查（需要 nginx-plus 或第三方模塊）
    # location @health {
    #     proxy_pass http://backend/health;
    # }
}

# ===== Nginx 日志配置 =====
http {
    # 自定義日志格式（包含更多調(diào)試信息）
    log_format detailed escape=json '{'
        '"time":"$time_iso8601",'
        '"remote_addr":"$remote_addr",'
        '"request":"$request",'
        '"status":"$status",'
        '"body_bytes_sent":"$body_bytes_sent",'
        '"request_time":"$request_time",'
        '"upstream_response_time":"$upstream_response_time",'
        '"upstream_connect_time":"$upstream_connect_time",'
        '"upstream_header_time":"$upstream_header_time",'
        '"http_referer":"$http_referer",'
        '"http_user_agent":"$http_user_agent",'
        '"upstream_addr":"$upstream_addr",'
        '"upstream_status":"$upstream_status"'
    '}';
    access_log /var/log/nginx/access.log detailed;
    error_log /var/log/nginx/error.log warn;
    # 單獨(dú)記錄上游錯(cuò)誤
    map $upstream_status $log_upstream_error {
        default 0;
        ~^5 1;  # 上游返回 5xx
    }
    server {
        access_log /var/log/nginx/access.log detailed;
        # 只記錄錯(cuò)誤
        location /api/ {
            proxy_pass http://backend;
            access_log /var/log/nginx/error.log combined if=$log_upstream_error;
        }
    }
}

#!/usr/bin/env python3
"""Nginx 日志分析腳本"""
import re
from collections import defaultdict, Counter
from datetime import datetime
import json
# 日志解析正則
LOG_PATTERN = re.compile(
    r'(?P<ip>\S+) \S+ \S+ \[(?P<time>[^\]]+)\] '
    r'"(?P<method>\S+) (?P<path>\S+) \S+" '
    r'(?P<status>\d+) (?P<size>\d+) '
    r'"(?P<referer>[^"]*)" "(?P<ua>[^"]*)"'
)
def parse_nginx_log(log_file):
    """解析 Nginx 日志"""
    entries = []
    with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
        for line in f:
            match = LOG_PATTERN.match(line)
            if match:
                entries.append(match.groupdict())
    return entries
def analyze_errors(log_file):
    """分析錯(cuò)誤日志"""
    entries = parse_nginx_log(log_file)
    # 按狀態(tài)碼統(tǒng)計(jì)
    status_counter = Counter(e['status'] for e in entries)
    # 按接口統(tǒng)計(jì)錯(cuò)誤
    error_by_path = defaultdict(Counter)
    for e in entries:
        if int(e['status']) >= 400:
            error_by_path[e['path']][e['status']] += 1
    # 按時(shí)間統(tǒng)計(jì)
    error_by_time = defaultdict(int)
    for e in entries:
        if int(e['status']) >= 400:
            # 提取小時(shí)
            time_str = e['time'].split(':')[0] + ':' + e['time'].split(':')[1]
            error_by_time[time_str] += 1
    # 打印結(jié)果
    print("=" * 60)
    print("狀態(tài)碼統(tǒng)計(jì)")
    print("=" * 60)
    for status, count in sorted(status_counter.items(), key=lambda x: -x[1]):
        print(f"  {status}: {count}")
    print("\n" + "=" * 60)
    print("錯(cuò)誤接口 Top 20")
    print("=" * 60)
    error_paths = [(path, sum(counter.values())) for path, counter in error_by_path.items()]
    error_paths.sort(key=lambda x: -x[1])
    for path, count in error_paths[:20]:
        status_dist = error_by_path[path]
        status_str = ", ".join(f"{s}:{c}" for s, c in status_dist.items())
        print(f"  {path}: {count} ({status_str})")
    print("\n" + "=" * 60)
    print("錯(cuò)誤時(shí)間分布（小時(shí)）")
    print("=" * 60)
    for time, count in sorted(error_by_time.items()):
        print(f"  {time}: {count}")
    return {
        'status': dict(status_counter),
        'error_by_path': {k: dict(v) for k, v in error_by_path.items()},
        'error_by_time': dict(error_by_time)
    }
def find_502_504_causes(log_file):
    """分析 502/504 原因"""
    with open(log_file, 'r', encoding='utf-8', errors='ignore') as f:
        content = f.read()
    # 查找連接拒絕
    connection_refused = len(re.findall(r'Connection refused', content))
    # 查找超時(shí)
    timed_out = len(re.findall(r'timed out', content))
    # 查找連接重置
    connection_reset = len(re.findall(r'Connection reset', content))
    print("\n" + "=" * 60)
    print("502/504 原因分析")
    print("=" * 60)
    print(f"  Connection refused: {connection_refused}")
    print(f"  Connection timed out: {timed_out}")
    print(f"  Connection reset: {connection_reset}")
    return {
        'connection_refused': connection_refused,
        'timed_out': timed_out,
        'connection_reset': connection_reset
    }
if __name__ == '__main__':
    import sys
    log_file = sys.argv[1] if len(sys.argv) > 1 else '/var/log/nginx/access.log'
    analyze_errors(log_file)
    find_502_504_causes(log_file)

#!/bin/bash
# ===== Nginx 實(shí)時(shí)監(jiān)控腳本 =====
LOG_FILE="/var/log/nginx/access.log"
echo "監(jiān)控 Nginx 錯(cuò)誤日志（按 Ctrl+C 停止）"
echo "========================================"
# 實(shí)時(shí)監(jiān)控 499/502/504
tail -f "$LOG_FILE" | grep --line-buffered -E " (499|502|504) "
# 或者更詳細(xì)的信息
# tail -f "$LOG_FILE" | grep --line-buffered -E " (499|502|504) " | awk '{print $1, $4, $7, $9}'

#!/bin/bash
# ===== Nginx 錯(cuò)誤統(tǒng)計(jì)腳本 =====
LOG_FILE="/var/log/nginx/access.log"
echo "Nginx 錯(cuò)誤統(tǒng)計(jì)報(bào)告"
echo "=================="
echo ""
# 總請(qǐng)求數(shù)
total=$(wc -l < "$LOG_FILE")
echo "總請(qǐng)求數(shù): $total"
# 各狀態(tài)碼統(tǒng)計(jì)
echo ""
echo "狀態(tài)碼分布："
awk '{print $9}' "$LOG_FILE" | sort | uniq -c | sort -rn | head -10
# 499 統(tǒng)計(jì)
echo ""
echo "499 錯(cuò)誤統(tǒng)計(jì)："
grep " 499 " "$LOG_FILE" | wc -l
echo ""
echo "499 錯(cuò)誤接口："
grep " 499 " "$LOG_FILE" | awk '{print $7}' | sort | uniq -c | sort -rn | head -10
# 502 統(tǒng)計(jì)
echo ""
echo "502 錯(cuò)誤統(tǒng)計(jì)："
grep " 502 " "$LOG_FILE" | wc -l
echo ""
echo "502 錯(cuò)誤接口："
grep " 502 " "$LOG_FILE" | awk '{print $7}' | sort | uniq -c | sort -rn | head -10
# 504 統(tǒng)計(jì)
echo ""
echo "504 錯(cuò)誤統(tǒng)計(jì)："
grep " 504 " "$LOG_FILE" | wc -l
echo ""
echo "504 錯(cuò)誤接口："
grep " 504 " "$LOG_FILE" | awk '{print $7}' | sort | uniq -c | sort -rn | head -10
# 慢請(qǐng)求（響應(yīng)時(shí)間 > 1s）
echo ""
echo "慢請(qǐng)求（>1s）統(tǒng)計(jì)："
awk '{print $NF, $0}' "$LOG_FILE" | awk '$1 > 1' | wc -l
echo ""
echo "最慢請(qǐng)求 Top 10："
awk '{print $NF, $7}' "$LOG_FILE" | sort -rn | head -10

# ===== 上游服務(wù)健康檢查 =====
# 1. 檢查服務(wù)是否運(yùn)行
systemctl status your-service
# 2. 檢查端口是否監(jiān)聽(tīng)
netstat -tlnp | grep 8000
ss -tlnp | grep 8000
# 3. 檢查進(jìn)程
ps aux | grep your-service
# 4. 測(cè)試連接
curl -I http://127.0.0.1:8000/health
# 5. 檢查連接數(shù)
ss -tan state established '( sport = :8000 )' | wc -l
# 6. 檢查 TIME_WAIT 數(shù)量
ss -tan state time-wait | wc -l
# 7. 檢查系統(tǒng)資源
free -h
df -h
top -bn1 | head -20
# 8. 檢查應(yīng)用日志
journalctl -u your-service -n 100
# 9. 檢查 Nginx 到上游的連接
ss -tan | grep :8000 | awk '{print $1}' | sort | uniq -c
# 10. 壓測(cè)上游服務(wù)
ab -n 1000 -c 10 http://127.0.0.1:8000/api/test

# ===== 案例：前端 30s 超時(shí)導(dǎo)致大量 499 =====

# 問(wèn)題現(xiàn)象
# Nginx 日志中大量 499，業(yè)務(wù)處理實(shí)際成功

# 排查步驟
# 1. 統(tǒng)計(jì) 499 分布
grep " 499 " /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn

# 輸出示例：
#    1523 /api/export-data
#     876 /api/batch-process
#     234 /api/heavy-compute

# 2. 分析這些接口的響應(yīng)時(shí)間
grep "/api/export-data" /var/log/nginx/access.log | awk '{print $NF}' | sort -n | tail -100

# 輸出示例：平均 45s

# 3. 檢查前端超時(shí)配置
# 前端 axios/requests 默認(rèn)超時(shí) 30s

# 原因分析
# - 后端處理需要 45s
# - 前端設(shè)置 30s 超時(shí)
# - 30s 后前端主動(dòng)斷開(kāi)，Nginx 返回 499

# 解決方案 1：增加前端超時(shí)
# axios.defaults.timeout = 60000; // 60s

# 解決方案 2：優(yōu)化后端處理
# - 使用異步處理 + 輪詢
# - 返回任務(wù) ID，前端輪詢狀態(tài)

# 解決方案 3：調(diào)整 Nginx 配置
location /api/export-data {
    proxy_pass http://backend;
    proxy_read_timeout 120s;  # 增加超時(shí)
}

# 最佳方案：異步處理
# 1. 接口立即返回 task_id
# 2. 后臺(tái)異步處理
# 3. 前端輪詢?nèi)蝿?wù)狀態(tài)
@app.post("/api/export")
async def export_data():
    task_id = await create_task()
    return {"task_id": task_id, "status": "pending"}

@app.get("/api/tasks/{task_id}")
async def get_task_status(task_id: str):
    return await get_task(task_id)

# ===== 案例：高峰期服務(wù)崩潰導(dǎo)致 502 =====

# 問(wèn)題現(xiàn)象
# 每天 10:00-12:00 高峰期大量 502

# 排查步驟
# 1. 分析 502 時(shí)間分布
grep " 502 " /var/log/nginx/access.log | awk '{print $4}' | cut -d: -f1-2 | sort | uniq -c

# 輸出示例：
#    15 2026/06/04:08
#    23 2026/06/04:09
#  1523 2026/06/04:10  <-- 高峰期
#  2345 2026/06/04:11  <-- 高峰期
#    12 2026/06/04:12

# 2. 檢查應(yīng)用日志
journalctl -u your-app --since "10:00" --until "12:00" | grep -i "error\|exception\|oom"

# 發(fā)現(xiàn)：OOM 錯(cuò)誤
# Jun 04 10:15:23 server your-app[12345]: java.lang.OutOfMemoryError: Java heap space

# 3. 檢查內(nèi)存使用
free -h

# 輸出：內(nèi)存幾乎用完

# 4. 檢查進(jìn)程內(nèi)存
ps aux --sort=-%mem | head -10

# 原因分析
# - 高峰期內(nèi)存不足
# - JVM OOM 導(dǎo)致服務(wù)崩潰
# - Nginx 無(wú)法連接，返回 502

# 解決方案
# 1. 增加 JVM 堆內(nèi)存
JAVA_OPTS="-Xms2g -Xmx4g"

# 2. 增加服務(wù)器內(nèi)存
# 或水平擴(kuò)展

# 3. 添加健康檢查
location /health {
    proxy_pass http://backend/health;
    proxy_connect_timeout 5s;
    proxy_read_timeout 5s;
}

# 4. Nginx 配置重試
location /api/ {
    proxy_pass http://backend;
    proxy_next_upstream error timeout http_502;
    proxy_next_upstream_tries 2;
}

# ===== 案例：數(shù)據(jù)庫(kù)慢查詢導(dǎo)致間歇性 504 =====

# 問(wèn)題現(xiàn)象
# 某些請(qǐng)求偶爾 504，不是持續(xù)發(fā)生

# 排查步驟
# 1. 分析 504 的請(qǐng)求特征
grep " 504 " /var/log/nginx/access.log | awk '{print $7}' | sort | uniq -c | sort -rn

# 輸出示例：
#    234 /api/orders
#    123 /api/reports
#     45 /api/search

# 2. 檢查這些接口的響應(yīng)時(shí)間分布
grep "/api/orders" /var/log/nginx/access.log | awk '{print $NF}' | sort -n | head -100
grep "/api/orders" /var/log/nginx/access.log | awk '{print $NF}' | sort -n | tail -100

# 發(fā)現(xiàn)：大部分 < 1s，少數(shù) > 60s

# 3. 檢查上游服務(wù)日志
grep "/api/orders" /var/log/app.log | grep -i "slow\|timeout"

# 發(fā)現(xiàn)：數(shù)據(jù)庫(kù)慢查詢
# [WARN] Slow query: 65.23s SELECT * FROM orders WHERE ...

# 原因分析
# - 特定查詢觸發(fā)慢查詢
# - 數(shù)據(jù)庫(kù)索引缺失
# - 數(shù)據(jù)量增長(zhǎng)導(dǎo)致性能下降

# 解決方案 1：優(yōu)化查詢
# 添加索引
CREATE INDEX idx_orders_user_created ON orders(user_id, created_at);

# 解決方案 2：增加 Nginx 超時(shí)（臨時(shí)）
location /api/orders {
    proxy_pass http://backend;
    proxy_read_timeout 120s;  # 增加到 2 分鐘
}

# 解決方案 3：添加緩存
location /api/orders {
    proxy_pass http://backend;
    proxy_cache orders_cache;
    proxy_cache_valid 200 5m;
    proxy_cache_key "$request_uri";
}

# 解決方案 4：讀寫(xiě)分離
# 讀請(qǐng)求走從庫(kù)

# ===== Prometheus Nginx 監(jiān)控配置 =====
# nginx-prometheus-exporter
# 或使用 nginx-module-vts
scrape_configs:
  - job_name: 'nginx'
    static_configs:
      - targets: ['localhost:9113']
# 告警規(guī)則
groups:
  - name: nginx-alerts
    rules:
      - alert: NginxHighErrorRate
        expr: |
          sum(rate(nginx_http_requests_total{status=~"5.."}[5m])) 
          / sum(rate(nginx_http_requests_total[5m])) > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Nginx 錯(cuò)誤率過(guò)高"
          description: "5xx 錯(cuò)誤率 {{ $value | humanizePercentage }}"
      - alert: NginxHigh499
        expr: |
          sum(rate(nginx_http_requests_total{status="499"}[5m])) 
          / sum(rate(nginx_http_requests_total[5m])) > 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Nginx 499 錯(cuò)誤率高"
          description: "客戶端取消率 {{ $value | humanizePercentage }}"
      - alert: NginxBackendDown
        expr: nginx_upstream_ups{status="down"} > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "上游服務(wù)不可用"
          description: "上游服務(wù) {{ $labels.upstream }} 處于 down 狀態(tài)"
      - alert: NginxHighLatency
        expr: histogram_quantile(0.99, rate(nginx_http_request_duration_seconds_bucket[5m])) > 5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Nginx 響應(yīng)延遲高"
          description: "P99 延遲 {{ $value }}s"

// Grafana Dashboard JSON（簡(jiǎn)化版）
{
  "panels": [
    {
      "title": "請(qǐng)求狀態(tài)碼分布",
      "type": "piechart",
      "targets": [
        {
          "expr": "sum by(status) (rate(nginx_http_requests_total[5m]))",
          "legendFormat": "{{status}}"
        }
      ]
    },
    {
      "title": "錯(cuò)誤率趨勢(shì)",
      "type": "graph",
      "targets": [
        {
          "expr": "sum(rate(nginx_http_requests_total{status=~\"5..\"}[5m]))",
          "legendFormat": "5xx"
        },
        {
          "expr": "sum(rate(nginx_http_requests_total{status=\"499\"}[5m]))",
          "legendFormat": "499"
        }
      ]
    },
    {
      "title": "上游響應(yīng)時(shí)間",
      "type": "heatmap",
      "targets": [
        {
          "expr": "rate(nginx_http_upstream_response_time_seconds_bucket[5m])",
          "format": "heatmap"
        }
      ]
    },
    {
      "title": "上游健康狀態(tài)",
      "type": "stat",
      "targets": [
        {
          "expr": "nginx_upstream_ups",
          "legendFormat": "{{upstream}}"
        }
      ]
    }
  ]
}