K8s 集群高可用操作问题记录
高可用的问题点就在于集群只能指定一个集群 ip，这个 ip 必需是负载均衡的 ip，或者网络内广播的 vip，云服务商提供的ecs一般都不支持广播，只能用负载均衡产品。 手上的负载均衡很难用，不是大厂的产品，https 转发只能用 http 请求目标主机，通过在主机添加 nginx http 转发到本机 6443 https 倒也能解决。 另一个问题，通过负载均衡到集群的流量会被标记为 system:anonymous，尝试通过 nginx 模拟负载均衡，nginx 6444 https 接收转发到本机 nginx 6442 http，再到 6443 https，无法复现。 只能换个思路处理了，开启 system:anonymous 访问权限，再限制所有控制节点 6443 端口只能内网访问。
system:anonymous 授权
kubectl create clusterrolebinding kubeadm-anonymous --clusterrole=cluster-admin --user=system:anonymous # 删除授权 kubectl delete clusterrolebinding kubeadm-anonymous
nginx 跟踪记录
# 配置添加日志打印
# 添加 log_format detailed*****
# 添加 access_log /var/log/nginx/lb-access.log detailed;
# 添加 access_log /var/log/nginx/direct-access.log detailed;
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
log_format detailed '$remote_addr - $remote_user [$time_local] '
'"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent" '
'"Authorization: $http_authorization" '
'X-Forwarded-Proto: $http_x_forwarded_proto '
'Host: $host';
access_log /var/log/nginx/access.log detailed;
error_log /var/log/nginx/error.log info;
##
# Gzip Settings
##
gzip on;
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
server {
listen 6442;
listen [::]:6442;
server_name _; # 不绑定特定域名，接受所有 HTTPS 请求
# ssl_certificate /etc/nginx/ssl/nginx.crt;
# ssl_certificate_key /etc/nginx/ssl/nginx.key;
location / {
proxy_pass https://127.0.0.1:6443; # 代理到后端服务
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header Authorization $http_authorization; # 确保转发 Authorization 头
proxy_buffering off; # 禁用缓冲，确保流式传输
proxy_redirect off;
access_log /var/log/nginx/lb-access.log detailed; # 负载均衡请求日志
}
}
server {
listen 6444 ssl;
server_name _; # 不绑定特定域名，接受所有 HTTPS 请求
ssl_certificate /etc/kubernetes/pki/apiserver.crt;
ssl_certificate_key /etc/kubernetes/pki/apiserver.key;
location / {
proxy_pass http://127.0.0.1:6442; # 代理到后端服务
proxy_set_header Host $host;
#proxy_set_header X-Real-IP $remote_addr;
#proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
#proxy_set_header X-Forwarded-Proto http;
# proxy_set_header Authorization $http_authorization; # 确保转发 Authorization 头
#proxy_buffering off; # 禁用缓冲，确保流式传输
#proxy_redirect off;
access_log /var/log/nginx/direct-access.log detailed; # 直接请求日志
}
}
nginx 日志
负载均衡： 6442： 172.22.8.51 - - [15/Mar/2025:11:03:14 +0800] "GET /api/v1/namespaces/kube-public/configmaps/cluster-info?timeout=10s HTTP/1.1" 200 2699 "-" "kubeadm/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:14 +0800] "GET /api/v1/namespaces/kube-public/configmaps/cluster-info?timeout=10s HTTP/1.1" 200 2699 "-" "kubeadm/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:14 +0800] "GET /api/v1/namespaces/kube-system/configmaps/kubeadm-config?timeout=10s HTTP/1.1" 200 1116 "-" "kubeadm/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: Bearer pym15t.dzssek8748pz666d" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:14 +0800] "GET /api/v1/namespaces/kube-system/configmaps/kube-proxy?timeout=10s HTTP/1.1" 200 2423 "-" "kubeadm/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: Bearer pym15t.dzssek8748pz666d" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:14 +0800] "GET /api/v1/namespaces/kube-system/configmaps/kubelet-config?timeout=10s HTTP/1.1" 200 1735 "-" "kubeadm/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: Bearer pym15t.dzssek8748pz666d" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:14 +0800] "GET /api/v1/nodes/lykj-ecs12-cs-1?timeout=10s HTTP/1.1" 404 200 "-" "kubeadm/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: Bearer pym15t.dzssek8748pz666d" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:15 +0800] "POST /apis/certificates.k8s.io/v1/certificatesigningrequests HTTP/1.1" 201 870 "-" "kubelet/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: Bearer pym15t.dzssek8748pz666d" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:15 +0800] "GET /api/v1/services?limit=500&resourceVersion=0 HTTP/1.1" 403 193 "-" "kubelet/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:15 +0800] "POST /api/v1/namespaces/default/events HTTP/1.1" 403 195 "-" "kubelet/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:15 +0800] "GET /api/v1/nodes?fieldSelector=metadata.name%3Dlykj-ecs12-cs-1&limit=500&resourceVersion=0 HTTP/1.1" 403 217 "-" "kubelet/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:15 +0800] "GET /apis/storage.k8s.io/v1/csinodes/lykj-ecs12-cs-1 HTTP/1.1" 403 269 "-" "kubelet/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:15 +0800] "GET /apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/lykj-ecs12-cs-1?timeout=10s HTTP/1.1" 403 292 "-" "kubelet/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:15 +0800] "GET /apis/certificates.k8s.io/v1/certificatesigningrequests?fieldSelector=metadata.name%3Dcsr-8qlwb HTTP/1.1" 200 892 "-" "kubelet/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: Bearer pym15t.dzssek8748pz666d" X-Forwarded-Proto: https Host: api.k8s.local 172.22.8.51 - - [15/Mar/2025:11:03:15 +0800] "GET /apis/node.k8s.io/v1/runtimeclasses?limit=500&resourceVersion=0 HTTP/1.1" 403 246 "-" "kubelet/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: https Host: api.k8s.local nginx模拟转发： 6444: 172.21.24.88 - - [15/Mar/2025:11:05:44 +0800] "GET /api/v1/namespaces/kube-public/configmaps/cluster-info?timeout=10s HTTP/1.1" 200 2699 "-" "kubeadm/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: - Host: api.k8s.local 172.21.24.88 - - [15/Mar/2025:11:05:44 +0800] "GET /api/v1/namespaces/kube-public/configmaps/cluster-info?timeout=10s HTTP/1.1" 200 2699 "-" "kubeadm/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: - Host: api.k8s.local 6442: 127.0.0.1 - - [15/Mar/2025:11:05:44 +0800] "GET /api/v1/namespaces/kube-public/configmaps/cluster-info?timeout=10s HTTP/1.0" 200 2687 "-" "kubeadm/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: http Host: api.k8s.local 127.0.0.1 - - [15/Mar/2025:11:05:44 +0800] "GET /api/v1/namespaces/kube-public/configmaps/cluster-info?timeout=10s HTTP/1.0" 200 2687 "-" "kubeadm/v1.28.15 (linux/amd64) kubernetes/8418565" "Authorization: -" X-Forwarded-Proto: http Host: api.k8s.local