前提

因为apiserver 是无状态话的，所以可以使用简单的4层负载均衡实现高可用！
下例子使用ha-proxy实现4层负载均衡和故障检查，使用keepalive实现虚拟地址。

创建haproxy的配置文件,文件名为：haproxy.cfg

  1. global
  2.     log         127.0.0.1 local2
  4.     pidfile     /var/run/haproxy.pid
  5.     maxconn     4000
  6.     daemon
  8. defaults
  9.     mode                    tcp
  10.     log                     global
  11.     option                  abortonclose 
  12.     option                  redispatch
  13.     retries                 3
  14.     timeout connect         10s
  15.     timeout client          1m
  16.     timeout server          1m
  17.     maxconn                 30000
  19. listen stats
  20.     mode http
  21.     bind 0.0.0.0:1080
  22.     stats enable
  23.     stats hide-version
  24.     stats uri     /haproxyadmin?stats
  25.     stats realm   Haproxy\ Statistics
  26.     stats auth    admin:123456
  27.     stats admin if TRUE
  29. listen check
  30.     mode http
  31.     bind 0.0:10200
  32.     stats enable
  33.     stats hide-version
  34.     stats uri /healthz
  35.     stats scope security_port 
  37. frontend security_port
  38.     bind *:8443
  39.     mode tcp
  40.     log global
  41.     default_backend servers
  44. backend servers
  45.     mode tcp
  46.     balance roundrobin
  47.     stick-table type ip size 200k expire 30m
  48.     stick on src
  49.     server master1 10.10.1.21:6443 weight 1 check inter 1000 rise 2 fall 2
  50.     server master2 10.10.1.22:6443 weight 1 check inter 1000 rise 2 fall 2
  51.     server master3 10.10.1.23:6443 weight 1 check inter 1000 rise 2 fall 2

创建keepalived 配置文件

  1. ! Configuration File for keepalived
  3. global_defs {
  4.    notification_email {
  5.      dblvs@jn.idc
  6.    }
  7.    notification_email_from Alexandre.Cassen@firewall.loc
  8.    smtp_server 127.0.0.1
  9.    smtp_connect_timeout 30
  10.    router_id LVS_DEVEL
  11. }
  13. vrrp_instance VI_1 {
  14.     state MASTER
  15.     interface ens33
  16.     virtual_router_id 51
  17.     priority 120
  18.     advert_int 1
  19.     authentication {
  20.         auth_type PASS
  21.         auth_pass 1111@2222
  22.     }
  23.     virtual_ipaddress {
  24.         10.10.10.20/24
  25.     }
  26. }
  28. virtual_server 10.10.10.20 8443 {
  29.     delay_loop 6
  30.     lb_algo wlc
  31.     lb_kind DR
  32.     nat_mask 255.255.255.0
  33.     #persistence_timeout 50
  34.     protocol TCP
  36.     real_server 10.10.10.21 8443 {
  37.         weight 1
  38.         TCP_CHECK {
  39.             connect_timeout 3
  40.             nb_get_retry 3
  41.             delay_before_retry 3
  42.         connect_port 8443
  43.         }
  44.     }
  45. }

创建 haproxy 和 keepalived 的staticPod文件 hub.k8s.com/apps/keepalived:1.4.1 这个镜像下载地址为 osixia/keepalived：1.4.1 ha-proxy.yaml

  1. apiVersion: v1
  2. kind: Pod
  3. metadata:
  4.   name: haproxy
  5.   namespace: kube-system
  6.   labels:
  7.     name: haproxy
  8. spec:
  9.   restartPolicy: Always
  10.   hostNetwork: true
  11.   containers:
  12.   - name: haproxy
  13.     image: hub.k8s.com/apps/haproxy:1.8.3
  14.     livenessProbe:
  15.       httpGet:
  16.         scheme: HTTP
  17.         host: 127.0.0.1
  18.         port: 10200
  19.         path: /healthz
  20.       initialDelaySeconds: 15
  21.       timeoutSeconds: 15
  22.     securityContext:
  23.       privileged: true
  24.     volumeMounts:
  25.     - name: lib
  26.       mountPath: /var/lib/haproxy
  27.       readOnly: false
  28.     - name: config
  29.       mountPath: /usr/local/etc/haproxy/
  30.       readOnly: false
  31.     ports:
  32.     - name: security-port
  33.       containerPort: 8443
  34.       protocol: TCP
  35.     - name: check-port
  36.       containerPort: 1080
  37.       protocol: TCP
  38.   - name: keepalived
  39.     image: hub.k8s.com/apps/keepalived:1.4.1
  40.     command:
  41.     - '/container/tool/run'
  42.     - '--copy-service'
  43.     securityContext:
  44.       privileged: true
  45.     volumeMounts:
  46.     - name: modules
  47.       mountPath: /lib/modules
  48.       readOnly: false
  49.     - name: kpconfig
  50.       mountPath: /container/service/keepalived/assets/keepalived.conf
  51.       readOnly: false
  52.   volumes:
  53.   - name: modules
  54.     hostPath:
  55.       path: /lib/modules/
  56.   - name: kpconfig
  57.     hostPath:
  58.       path: /opt/keepalived/keepalived.conf
  59.   - name: lib
  60.     emptyDir: {}
  61.   - name: config
  62.     hostPath:
  63.       path: /opt/haproxy/