| 主机名 | IP | 角色 | 主要插件 |
|---|---|---|---|
| VIP | 172.16.1.10 | 实现master高可用和负载均衡 | |
| k8s-master01 | 172.16.1.11 | master | kube-apiserver、kube-controller、kube-scheduler、kubelet、kube-proxy、kube-flannel、etcd |
| k8s-master02 | 172.16.1.12 | master | kube-apiserver、kube-controller、kube-scheduler、kubelet、kube-proxy、kube-flannel、etcd |
| k8s-master03 | 172.16.1.13 | master | kube-apiserver、kube-controller、kube-scheduler、kubelet、kube-proxy、kube-flannel、etcd |
| k8s-node01 | 172.16.1.21 | node | kubelet、kube-proxy、kube-flannel |
| k8s-node02 | 172.16.1.22 | node | kubelet、kube-proxy、kube-flannel |
master节点通过keepalived和haproxy来实现的高可用和负载均衡，对于云主机可以直接使用相关云产品，例如阿里云的slb或者腾讯云的clb。
在所有节点上作如下准备
建议至少2 CPU 、2G，非硬性要求，1CPU、1G也可以搭建起集群，但是在部署时会有WARNING提示：
#1个CPU的初始化master的时候会报
[WARNING NumCPU]: the number of available CPUs 1 is less than the required 2
#部署插件或者pod时可能会报
warning：FailedScheduling：Insufficient cpu, Insufficient memorycat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.netfilter.nf_conntrack_max = 2310720
fs.inotify.max_user_watches=89100
fs.may_detach_mounts = 1
fs.file-max = 52706963
fs.nr_open = 52706963
net.bridge.bridge-nf-call-arptables = 1
vm.swappiness = 0 #最大限度使用物理内存，然后才是 swap空间
vm.overcommit_memory=1
vm.panic_on_oom=0
EOF
sysctl --systemk8s1.8版本以后，要求关闭swap，否则默认配置下kubelet将无法启动。
#临时关闭
swapoff -a
#永久关闭
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstabmodprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
#查看是否加载
lsmod | grep ip_vs
#配置开机自加载
cat <<EOF>> /etc/rc.local
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod +x /etc/rc.d/rc.local#临时关闭
setenforce 0
#永久关闭
sed -ir 's/(SELINUX=)[a-z]*/\1diabled/' /etc/selinux/configsystemctl stop firewalld
systemctl disable firewalld#获取docker-ce的yum源
wget -O /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#获取epel源
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum -y install epel-release
#安装docker
yum -y install docker-ce
docker version
systemctl start docker
systemctl enable docker#注意
这里安装没有指定docker-ce版本，默认安装最新版，如果需要安装指定版本，可以：
#列出docker-ce版本
yum list docker-ce --showduplicates
#安装指定版本
yum -y install docker-ce-<VERSION_STRING>ssh免密登录、hosts文件、ntp时间同步
master节点执行
yum install -y socat keepalived haproxy ipvsadm
systemctl enable haproxy
systemctl enable keepalivedhaproxy配置文件：
#/etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local3
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 32768
user haproxy
group haproxy
daemon
nbproc 1
stats socket /var/lib/haproxy/stats
defaults
mode tcp
log global
option tcplog
option dontlognull
option redispatch
retries 3
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout check 10s
listen stats
mode http
bind :8888
stats enable
stats uri /admin?stats
stats auth admin:admin
stats admin if TRUE
frontend k8s_https *:8443
mode tcp
maxconn 2000
default_backend https_sri
backend https_sri
balance roundrobin
server master1-api 172.16.1.11:6443 check inter 10000 fall 2 rise 2 weight 1
server master2-api 172.16.1.12:6443 check inter 10000 fall 2 rise 2 weight 1
server master3-api 172.16.1.13:6443 check inter 10000 fall 2 rise 2 weight 1keepalived配置文件：
#/etc/keepalived/keepalived.conf
global_defs {
router_id master01
}
vrrp_script check_haproxy {
script /etc/keepalived/check_haproxy.sh
interval 3
}
vrrp_instance VI_1 {
state MASTER
interface ens192
virtual_router_id 80
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
172.16.1.10/24
}
track_script {
check_haproxy
}
}
}#/etc/keepalived/check_haproxy.sh
#!/bin/bash
NUM=`ps -C haproxy --no-header |wc -l`
if [ $NUM -eq 0 ];then
systemctl stop keepalived
fi注意，三个节点keepalived配置文件存在区别：
router_id分别为master01、master02、master03
state分别为MASTER、BACKUP、BACKUP
priority分别为100、90、80
所有节点都安装 kubeadm、kubelet、kubectl，注意：node节点的kubectl不是必须的。
#配置yum源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpghttps://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
#安装
yum -y install kubeadm-1.14.0 kubelet-1.14.0 kubectl-1.14.0
systemctl enable kubelet通过kubeadm config print init-defaults > kubeadm.conf可以获得默认配置文件。
#查看需要的镜像
kubeadm config images list --config kubeadm.conf
#拉取需要的镜像
kubeadm config images pull --config kubeadm.conf
#初始化
kubeadm init –config kubeadm.conf配置文件kubeadm_master01.conf
apiVersion: kubeadm.k8s.io/v1beta1
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 172.16.1.11
bindPort: 6443
---
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: v1.14.0
controlPlaneEndpoint: "172.16.1.10:8443"
imageRepository: registry.aliyuncs.com/google_containers
apiServer:
certSANs:
- "k8s-master01"
- "k8s-master02"
- "k8s-master03"
- 172.16.1.11
- 172.16.1.12
- 172.16.1.13
- 172.16.1.10
networking:
podSubnet: "10.244.0.0/16"
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
etcd:
local:
extraArgs:
listen-client-urls: "https://127.0.0.1:2379,https://172.16.1.11:2379"
advertise-client-urls: "https://172.16.1.11:2379"
listen-peer-urls: "https://172.16.1.11:2380"
initial-advertise-peer-urls: "https://172.16.1.11:2380"
initial-cluster: "k8s-master01=https://172.16.1.11:2380"
initial-cluster-state: new
serverCertSANs:
- k8s-master01
- 172.16.1.11
peerCertSANs:
- k8s-master01
- 172.16.1.11
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs#初始化master01
kubeadm init --config kubeadm_master01.conf
#配置kubectl管理集群
mkdir .kube
cp -i /etc/kubernetes/admin.conf .kube/config而后才可以执行kubectl相关命令，例如查看当前存在pod，可以发现唯独coredns的pod是出于Pending状态，原因是还未安装网络插件。
[root@k8s-master01 ~]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-8686dcc4fd-96lr9 0/1 Pending 0 40m
coredns-8686dcc4fd-xk9st 0/1 Pending 0 40m
etcd-k8s-master01 1/1 Running 0 39m
kube-apiserver-k8s-master01 1/1 Running 0 39m
kube-controller-manager-k8s-master01 1/1 Running 0 39m
kube-proxy-2cb7r 1/1 Running 0 40m
kube-scheduler-k8s-master01 1/1 Running 0 39m
#安装Flannel网络插件
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml#绑定网卡
flannel 默认会使用主机的第一张网卡，如果你有多张网卡，需要指定时，可以修改 kube-flannel.yml 中的以下部分
containers:
- name: kube-flannel
image: quay.io/coreos/flannel:v0.11.0-amd64
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
- --iface=ens192 #添加该行
#而后应用配置文件（注意只在master01执行）
kubectl apply -f kube-flannel.yml#此时，我们再查看pod发现coredns已不是Pending状态
[root@k8s-master01 ~]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-8686dcc4fd-96lr9 1/1 Running 0 54m
coredns-8686dcc4fd-xk9st 1/1 Running 0 54m
etcd-k8s-master01 1/1 Running 0 53m
kube-apiserver-k8s-master01 1/1 Running 0 53m
kube-controller-manager-k8s-master01 1/1 Running 0 53m
kube-flannel-ds-amd64-4vg2s 1/1 Running 0 50s
kube-proxy-2cb7r 1/1 Running 0 54m
kube-scheduler-k8s-master01 1/1 Running 0 53m执行如下脚本
#!/bin/bash
for index in 12 13; do
ip=172.16.1.${index}
ssh $ip "mkdir -p /etc/kubernetes/pki/etcd; mkdir -p ~/.kube/"
scp /etc/kubernetes/pki/ca.crt $ip:/etc/kubernetes/pki/ca.crt
scp /etc/kubernetes/pki/ca.key $ip:/etc/kubernetes/pki/ca.key
scp /etc/kubernetes/pki/sa.key $ip:/etc/kubernetes/pki/sa.key
scp /etc/kubernetes/pki/sa.pub $ip:/etc/kubernetes/pki/sa.pub
scp /etc/kubernetes/pki/front-proxy-ca.crt $ip:/etc/kubernetes/pki/front-proxy-ca.crt
scp /etc/kubernetes/pki/front-proxy-ca.key $ip:/etc/kubernetes/pki/front-proxy-ca.key
scp /etc/kubernetes/pki/etcd/ca.crt $ip:/etc/kubernetes/pki/etcd/ca.crt
scp /etc/kubernetes/pki/etcd/ca.key $ip:/etc/kubernetes/pki/etcd/ca.key
scp /etc/kubernetes/admin.conf $ip:/etc/kubernetes/admin.conf
scp /etc/kubernetes/admin.conf $ip:~/.kube/config
done配置文件kubeadm_master02.conf
apiVersion: kubeadm.k8s.io/v1beta1
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 172.16.1.12
bindPort: 6443
---
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: v1.14.0
controlPlaneEndpoint: "172.16.1.10:8443"
imageRepository: registry.aliyuncs.com/google_containers
apiServer:
certSANs:
- "k8s-master01"
- "k8s-master02"
- "k8s-master03"
- 172.16.1.11
- 172.16.1.12
- 172.16.1.13
- 172.16.1.10
networking:
podSubnet: "10.244.0.0/16"
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
etcd:
local:
extraArgs:
listen-client-urls: "https://127.0.0.1:2379,https://172.16.1.12:2379"
advertise-client-urls: "https://172.16.1.12:2379"
listen-peer-urls: "https://172.16.1.12:2380"
initial-advertise-peer-urls: "https://172.16.1.12:2380"
initial-cluster: "k8s-master01=https://172.16.1.11:2380,k8s-master02=https://172.16.1.12:2380"
initial-cluster-state: existing
serverCertSANs:
- k8s-master02
- 172.16.1.12
peerCertSANs:
- k8s-master02
- 172.16.1.12
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs#配置证书
kubeadm init phase certs all --config kubeadm_master02.conf
#配置etcd
kubeadm init phase etcd local --config kubeadm_master02.conf
#生成kubelet配置文件
kubeadm init phase kubeconfig kubelet --config kubeadm_master02.conf
#启动kubelet
kubeadm init phase kubelet-start --config kubeadm_master02.conf
#将master02的etcd加入集群
kubectl exec -n kube-system etcd-k8s-master01 -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://172.16.1.11:2379 member add master2 https://172.16.1.12:2380
#启动 kube-apiserver、kube-controller-manager、kube-scheduler
kubeadm init phase kubeconfig all --config kubeadm_master02.conf
kubeadm init phase control-plane all --config kubeadm_master02.conf#查看节点状态
[root@k8s-master02 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 3h3m v1.14.0
k8s-master02 Ready <none> 14m v1.14.0#将节点标记为master
kubeadm init phase mark-control-plane --config kubeadm_master02.conf#再次查看
[root@k8s-master02 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 3h4m v1.14.0
k8s-master02 Ready master 16m v1.14.0配置文件kubeadm_master03.conf
apiVersion: kubeadm.k8s.io/v1beta1
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 172.16.1.13
bindPort: 6443
---
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
kubernetesVersion: v1.14.0
controlPlaneEndpoint: "172.16.1.10:8443"
imageRepository: registry.aliyuncs.com/google_containers
apiServer:
certSANs:
- "k8s-master01"
- "k8s-master02"
- "k8s-master03"
- 172.16.1.11
- 172.16.1.12
- 172.16.1.13
- 172.16.1.10
networking:
podSubnet: "10.244.0.0/16"
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
etcd:
local:
extraArgs:
listen-client-urls: "https://127.0.0.1:2379,https://172.16.1.13:2379"
advertise-client-urls: "https://172.16.1.13:2379"
listen-peer-urls: "https://172.16.1.13:2380"
initial-advertise-peer-urls: "https://172.16.1.13:2380"
initial-cluster: "k8s-master01=https://172.16.1.11:2380,k8s-master02=https://172.16.1.12:2380,k8s-master03=https://172.16.1.13:2380"
initial-cluster-state: existing
serverCertSANs:
- k8s-master03
- 172.16.1.13
peerCertSANs:
- k8s-master03
- 172.16.1.13
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs#配置证书
kubeadm init phase certs all --config kubeadm_master03.conf
#配置etcd
kubeadm init phase etcd local --config kubeadm_master03.conf
#生成kubelet配置文件
kubeadm init phase kubeconfig kubelet --config kubeadm_master03.conf
#启动kubelet
kubeadm init phase kubelet-start --config kubeadm_master03.conf
#将master03的etcd加入集群
kubectl exec -n kube-system etcd-k8s-master01 -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://172.16.1.11:2379 member add master3 https://172.16.1.13:2380
#启动 kube-apiserver、kube-controller-manager、kube-scheduler
kubeadm init phase kubeconfig all --config kubeadm_master03.conf
kubeadm init phase control-plane all --config kubeadm_master03.conf
#将节点标记为master
kubeadm init phase mark-control-plane --config kubeadm_master03.conf通过以上步骤，三台master已初始化完毕。
#初始化master01时提示如下命令：
kubeadm join 172.16.1.10:8443 --token 8j5lga.y2cei06i6cfxbxmo \
--discovery-token-ca-cert-hash sha256:9eff14803a65631b74e4db6dfa9e7362eb1dd62cd76d56e840d33b1f5a3aa93b#查看node信息
[root@k8s-master01 ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master01 Ready master 3h29m v1.14.0
k8s-master02 Ready master 114m v1.14.0
k8s-master03 Ready master 95m v1.14.0
k8s-node01 Ready <none> 64m v1.14.0
k8s-node02 Ready <none> 50m v1.14.0#查看集群信息
[root@k8s-master01 ~]# kubectl cluster-info
Kubernetes master is running at https://172.16.1.10:8443
KubeDNS is running at https://172.16.1.10:8443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.#查看控制器状态
[root@k8s-master01 ~]# kubectl get cs
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-0 Healthy {"health":"true"} #查看etcd集群成员信息
[root@k8s-master01 ~]# kubectl exec -n kube-system etcd-k8s-master01 -- etcdctl --ca-file /etc/kubernetes/pki/etcd/ca.crt --cert-file /etc/kubernetes/pki/etcd/peer.crt --key-file /etc/kubernetes/pki/etcd/peer.key --endpoints=https://172.16.1.11:2379 member list
2cd4d60db6db4371: name=k8s-master01 peerURLs=https://172.16.1.11:2380 clientURLs=https://172.16.1.11:2379 isLeader=true
707da0ac9cb69832: name=k8s-master02 peerURLs=https://172.16.1.12:2380 clientURLs=https://172.16.1.12:2379 isLeader=false
c702920d32ced638: name=k8s-master03 peerURLs=https://172.16.1.13:2380 clientURLs=https://172.16.1.13:2379 isLeader=false#检查ipvs是否启用
通过ipvsadm可以看到规则
[root@k8s-master01 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.96.0.1:443 rr
-> 172.16.1.11:6443 Masq 1 0 0
-> 172.16.1.12:6443 Masq 1 0 0
-> 172.16.1.13:6443 Masq 1 1 0
TCP 10.96.0.10:53 rr
-> 10.244.3.2:53 Masq 1 0 0
-> 10.244.4.2:53 Masq 1 0 0
TCP 10.96.0.10:9153 rr
-> 10.244.3.2:9153 Masq 1 0 0
-> 10.244.4.2:9153 Masq 1 0 0
UDP 10.96.0.10:53 rr
-> 10.244.3.2:53 Masq 1 0 0
-> 10.244.4.2:53 Masq 1 0 0 通过kubectl log --tail=10 kube-proxy-tqxlq -n kube-system，查看pod日志，可以看到：Using ipvs Proxier.
如果ipvsadm检查不到规则，而且kube-proxy日志中发现：
can't determine whether to use ipvs proxy, error: IPVS proxier will not be used because the following required kernel modules are not loaded: [ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh]
Using iptables Proxier.说明ipvs启用失败。
免责声明：本站发布的内容（图片、视频和文字）以原创、转载和分享为主，文章观点不代表本网站立场，如果涉及侵权请联系站长邮箱：is@yisu.com进行举报，并提供相关证据，一经查实，将立刻删除涉嫌侵权内容。
