$ kubectl run test --image=nginx
Error from server (Forbidden): error when creating "test": pods "test" is forbidden: 
etcdserver: request timed out

$ journalctl -u etcd --no-pager -n 50
etcdserver: mvcc: database space exceeded

# 查看事件數(shù)量
$ kubectl get events --all-namespaces | wc -l
1234567

# 臨時(shí)擴(kuò)大 etcd 配額（從 2GB 擴(kuò)到 8GB）
$ etcdctl --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  put quota/bytes -- '8589934592'

# 刪除所有 namespace 下的舊事件（保留最近 1 小時(shí)的）
$ kubectl get events --all-namespaces --field-selector eventTimestamp<2026-04-27T09:00:00Z \
  | awk '{print $1}' | sort -u | xargs -I {} kubectl delete events --field-selector eventTimestamp<2026-04-27T09:00:00Z -n {}

# 在 kube-controller-manager 配置中設(shè)置事件 TTL
$ vi /etc/kubernetes/manifests/kube-controller-manager.yaml
spec:
  containers:
  - command:
    - kube-controller-manager
    - --terminated-pod-gc-threshold=1000
    - --event-ttl=1h    # 事件保留 1 小時(shí)后自動(dòng)清理

$ kubectl exec -it dev-app-pod -- dig dev-service.default.svc.cluster.local

;; Query time: 12 msec

$ kubectl exec -it dev-app-pod -- dig dev-service.default.svc.cluster.local.xxx

;; Query time: 3125 msec

$ kubectl exec -it dev-app-pod -- cat /etc/resolv.conf
nameserver 10.96.0.10
search default.svc.cluster.local svc.cluster.local cluster.local
options ndots:5

1. dev-service.default.svc.cluster.local   → NXDOMAIN
2. dev-service.svc.cluster.local           → NXDOMAIN
3. dev-service.cluster.local               → NXDOMAIN
4. dev-service（原始域名）                 → 成功

$ kubectl edit configmap coredns -n kube-system
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health
        kubernetes cluster.local in-addr.arpa ip6.arpa {
            pods insecure
            fallthrough in-addr.arpa ip6.arpa
            ttl 30
        }
        prometheus :9153
        forward . /etc/resolv.conf {
            max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }

spec:
  dnsConfig:
    options:
      - name: ndots
        value: "2"

spec:
  template:
    spec:
      dnsConfig:
        options:
          - name: ndots
            value: "2"

$ kubectl create secret generic api-secret \
  --from-literal=api-key=new-key-12345 \
  --dry-run=client -o yaml | kubectl apply -f -

env:
  - name: API_KEY
    valueFrom:
      secretKeyRef:
        name: api-secret
        key: api-key

volumes:
  - name: secret-volume
    secret:
      secretName: api-secret
volumeMounts:
  - name: secret-volume
    mountPath: /etc/secrets
    readOnly: true

$ kubectl rollout restart deployment app-service

spec:
  containers:
  - name: dev-app
    volumeMounts:
    - name: secret-volume
      mountPath: /etc/secrets
      readOnly: true
    env:
    - name: SECRET_PATH
      value: "/etc/secrets/api-key"
  volumes:
  - name: secret-volume
    secret:
      secretName: api-secret

// Go 示例：監(jiān)聽 Secret 文件變化
watcher, _ := fsnotify.NewWatcher("/etc/secrets/api-key")
go func() {
    for {
        select {
        case event := <-watcher.Events:
            if event.Op&fsnotify.Write == fsnotify.Write {
                newKey, _ := os.ReadFile("/etc/secrets/api-key")
                reloadAPIKey(string(newKey))
            }
        }
    }
}()

$ kubectl get hpa
NAME         REFERENCE                       TARGETS   MINPODS   MAXPODS   REPLICAS
dev-service   Deployment/dev-service/Scale   85%/80%   3         20        8

$ kubectl get pods | grep dev-service
dev-service-xxx-abc12   1/1   Running   0          15m
dev-service-xxx-def34   1/1   Running   0          15m
dev-service-xxx-ghi56   1/1   Running   0          15m
dev-service-xxx-jkl78   0/1   Pending   0          5m
dev-service-xxx-mno90   0/1   Pending   0          5m
dev-service-xxx-pqr12   0/1   Pending   0          5m
dev-service-xxx-stu34   0/1   Pending   0          5m
dev-service-xxx-vwx56   0/1   Pending   0          5m

$ kubectl describe pod dev-service-xxx-jkl78 | tail -10
Events:
  Type     Reason            Age   From               Message
  ----     ------            ----  ----               -------
  Warning  FailedScheduling  5m    default-scheduler   0/5 nodes are available: 3 Insufficient cpu, 2 Insufficient memory.

resources:
  requests:
    cpu: "100m"
    memory: "128Mi"
  limits:
    cpu: "2"
    memory: "4Gi"

# 查看哪些 Pod 占用資源最多
$ kubectl top pods --all-namespaces | sort -k3 -rn | head -20

# 臨時(shí)縮容非核心服務(wù)
$ kubectl scale deployment non-critical-service --replicas=0

# 查看 Pod 歷史資源使用量（需要 metrics-server）
$ kubectl top pod dev-service-xxx-abc12 --no-headers
# 輸出類似：523m 384Mi

# 建議取 P50 或 P70 的值作為 request
# 如果 P50 CPU = 500m，P70 CPU = 800m
# 那 request 設(shè)為 600m-800m 比較合理
resources:
  requests:
    cpu: "800m"      # 接近實(shí)際使用量
    memory: "512Mi"
  limits:
    cpu: "2000m"
    memory: "2Gi"

$ kubectl drain worker-03 --ignore-daemonsets --delete-emptydir-data
node/worker-03 cordoned
evicting pod default/my-service-xxx-abc12
evicting pod kube-system/calico-node-xxx
...

# 很多 Pod 的 terminationGracePeriodSeconds 用的是默認(rèn)值 30 秒
# 但應(yīng)用沒有監(jiān)聽 SIGTERM 信號(hào)，不會(huì)主動(dòng)退出

spec:
  terminationGracePeriodSeconds: 60   # 給足夠的時(shí)間
  containers:
  - name: dev-app
    lifecycle:
      preStop:
        exec:
          command: ["/bin/sh", "-c", "sleep 10"]  # 預(yù)停止鉤子，先從 Service 摘除
    # 應(yīng)用本身需要監(jiān)聽 SIGTERM 信號(hào)

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: dev-service-pdb
spec:
  minAvailable: "50%"
  selector:
    matchLabels:
      app: dev-service

$ kubectl drain worker-03 --ignore-daemonsets
error when evicting pods/default/my-service-xxx: 
Cannot evict pod as it would violate the pod's disruption budget.

# 正確做法：一個(gè)一個(gè)來(lái)
$ kubectl drain worker-03 --ignore-daemonsets --delete-emptydir-data
# 等所有 Pod 都遷移完成
$ kubectl get nodes worker-03
# 確認(rèn) Ready 后再操作下一個(gè)
$ kubectl drain worker-04 --ignore-daemonsets --delete-emptydir-data

$ top -bn1 | head -20
  PID USER      PR  NI    VIRT    RES    SHR S  %CPU  %MEM     TIME+ COMMAND
 5678 root      20   0  512.4m  128.6m  12.2m S  95.3   1.6  2345:12.78 fluentd
 5679 root      20   0  498.2m  115.3m  10.8m S  92.1   1.4  2198:34.56 fluentd

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: kube-system
spec:
  template:
    spec:
      containers:
      - name: fluentd
        resources:
          requests:
            cpu: "100m"
            memory: "200Mi"
          limits:
            cpu: "1000m"     # 限制最大 CPU
            memory: "1Gi"

# 找到日志量最大的 Pod
$ kubectl top pods --all-namespaces | sort -k3 -rn | head -10

# 修改日志級(jí)別
$ kubectl set env deployment dev-service LOG_LEVEL=INFO

# Fluentd 配置
<source>
  @type tail
  path /var/log/containers/*.log
  pos_file /var/log/fluentd-containers.log.pos
  tag kubernetes.*
  read_from_head true
  <parse>
    @type json
    time_key time
    time_format %Y-%m-%dT%H:%M:%S.%NZ
  </parse>
</source>
# 限制采集速率
<filter kubernetes.**>
  @type throttle
  group_key log
  group_bucket_limit 100    # 每秒最多采集 100 條
  group_interval 10s
  group_reset_rate 10/m
</filter>

$ kubectl set image deployment/dev-service dev-app=evd-app:v2.0.0
deployment.apps/dev-service image updated

JVM 啟動(dòng) → Spring 初始化 → 連接數(shù)據(jù)庫(kù) → 加載緩存 → 開始監(jiān)聽端口

readinessProbe:
  tcpSocket:
    port: 8080
  initialDelaySeconds: 10
  periodSeconds: 5

readinessProbe:
  httpGet:
    path: /health/ready    # 應(yīng)用提供一個(gè)真正的就緒檢查接口
    port: 8080
  initialDelaySeconds: 15
  periodSeconds: 5
  failureThreshold: 3

@GetMapping("/health/ready")
public ResponseEntity<String> ready() {
    // 檢查數(shù)據(jù)庫(kù)連接池是否就緒
    if (!dataSource.isRunning()) {
        return ResponseEntity.status(503).body("database not ready");
    }
    // 檢查緩存是否加載完成
    if (!cacheManager.isReady()) {
        return ResponseEntity.status(503).body("cache not ready");
    }
    return ResponseEntity.ok("ok");
}

spec:
  terminationGracePeriodSeconds: 60
  containers:
  - name: my-app
    lifecycle:
      preStop:
        exec:
          command: ["/bin/sh", "-c", "sleep 10"]

t=0s    preStop sleep 10 開始執(zhí)行
t=0s    endpoint controller 從 endpoint 列表中移除舊 Pod
t=10s   preStop 執(zhí)行完畢，kubelet 發(fā)送 SIGTERM
t=10s   應(yīng)用收到 SIGTERM，開始優(yōu)雅關(guān)閉
t=40s   應(yīng)用完成清理，主動(dòng)退出
t=60s   terminationGracePeriodSeconds 到期，強(qiáng)制 SIGKILL

spec:
  strategy:
    rollingUpdate:
      maxSurge: 1           # 滾動(dòng)更新時(shí)最多多創(chuàng)建 1 個(gè) Pod
      maxUnavailable: 0     # 滾動(dòng)更新時(shí)不允許有 Pod 不可用