kubeadm k8s HA 高可用集群搭建

发表于

修改host文件

1
2
3
4
5
cat <</etc/hosts>>EOF
172.16.70.251 master-1
172.16.70.252 master-2
172.16.70.1 master-3
EOF

安装docker-ce

1
2
3
4
5
sudo yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sudo yum makecache fast
sudo yum -y update
sudo yum install docker-ce -y
sudo systemctl start docker

配置阿里云k8syum仓库

1
2
3
4
5
6
7
8
9
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

初始化环境，升级内核

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
systemctl disable --now firewalld NetworkManager
setenforce 0
sed -ri '/^[^#]*SELINUX=/s#=.+$#=disabled#' /etc/selinux/config
systemctl disable --now dnsmasq
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
yum install epel-release -y
yum install wget git jq psmisc -y
yum update -y
[ ! -f /usr/bin/perl ] && yum install perl -y
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm
export Kernel_Version=4.18.9-1
wget http://mirror.rc.usf.edu/compute_lock/elrepo/kernel/el7/x86_64/RPMS/kernel-ml{,-devel}-${Kernel_Version}.el7.elrepo.x86_64.rpm
yum localinstall -y kernel-ml*
find /lib/modules -name '*nf_conntrack_ipv4*' -type f
grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --default-kernel
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"
reboot

安装IPVS

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
yum install ipvsadm ipset sysstat conntrack libseccomp -y
:> /etc/modules-load.d/ipvs.conf
module=(
  ip_vs
  ip_vs_lc
  ip_vs_wlc
  ip_vs_rr
  ip_vs_wrr
  ip_vs_lblc
  ip_vs_lblcr
  ip_vs_dh
  ip_vs_sh
  ip_vs_fo
  ip_vs_nq
  ip_vs_sed
  ip_vs_ftp
  )
for kernel_module in ${module[@]};do
    /sbin/modinfo -F filename $kernel_module |& grep -qv ERROR && echo $kernel_module >> /etc/modules-load.d/ipvs.conf || :
done
systemctl enable --now systemd-modules-load.service

cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
EOF
sysctl --system

安装docker补全

1
2
yum install -y epel-release bash-completion && cp /usr/share/bash-completion/completions/docker /etc/bash_completion.d/
systemctl enable --now docker

安装k8s组件

1
yum -y install kubeadm kubelet kubectl

拉取k8s镜像

1
kubeadm config images pull --config kubeadm-config.yaml

kubeadm初始化

1
kubeadm init --config kubeadm-config.yml

copy master-1证书到 master-2 master-3

1
2
3
4
5
6
7
8
9
10
11
12
13
USER=root
CONTROL_PLANE_IPS="master-2 master-3"
for host in ${CONTROL_PLANE_IPS}; do
    scp /etc/kubernetes/pki/ca.crt "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/ca.key "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/sa.key "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/sa.pub "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/front-proxy-ca.crt "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/pki/front-proxy-ca.key "${USER}"@$host:/etc/kubernetes/pki/
    scp /etc/kubernetes/admin.conf "${USER}"@$host:/etc/kubernetes/
    scp /etc/kubernetes/pki/etcd/ca.crt "${USER}"@$host:/etc/kubernetes/pki/etcd/ca.crt
    scp /etc/kubernetes/pki/etcd/ca.key "${USER}"@$host:/etc/kubernetes/pki/etcd/ca.key
done

安装calico

1
2
3
4
5
6
curl -s https://zhangguanzhang.github.io/bash/pull.sh | bash -s -- quay.io/calico/node:v3.4.0
curl -s https://zhangguanzhang.github.io/bash/pull.sh | bash -s -- quay.io/calico/cni:v3.4.0
curl -s https://zhangguanzhang.github.io/bash/pull.sh | bash -s -- quay.io/calico/kube-controllers:v3.4.0
curl -s https://zhangguanzhang.github.io/bash/pull.sh | bash -s -- quay.io/coreos/etcd:v3.3.9
kubectl apply -f https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/etcd.yaml
kubectl apply -f https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml

安装metrics-server

  1. 拉取yaml文件

    1
    2
    git clone https://github.com/kubernetes-incubator/metrics-server.git
    vim deploy/1.8+/metrics-server-deployment.yaml

  2. 增加 spec.template.spec.containers.command:

    1
    2
    3
    4
    5
    command:
    - /metrics-server
    - --kubelet-insecure-tls
    - --kubelet-preferred-address-types=InternalIP
    imagePullPolicy: IfNotPresent  ## 修改 imagePullPolicy: IfNotPresent 否咋镜像拉取不到

  3. 拉取镜像

    1
    curl -s https://zhangguanzhang.github.io/bash/pull.sh | bash -s -- k8s.gcr.io/metrics-server-amd64:v0.3.1

  4. 创建metrics-server

    1
    kubectl create -f deploy/1.8+/