#yyds干货盘点#--kubernetes集群搭建

#yyds干货盘点#--kubernetes集群搭建

一、环境说明

二、系统初始化

2.1 配置主机名添加hosts解析

cat <>/etc/hosts10.131.100.92 k8s-master0110.131.100.93 k8s-master0210.131.100.94 k8s-master0310.131.100.95 k8s-node01EOF

2.2 关闭防火墙、selinux

systemctl stop firewalldsystemctl disable firewalldsetenforce 0sed -i "s/^SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config

2.3 配置内核参数，将桥接的IPv4流量传递到iptables的链

cat > /etc/sysctl.d/k8s.conf <

2.4 配置国内yum源

yum install -y wgetmkdir /etc/yum.repos.d/bak && mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/bakwget -O /etc/yum.repos.d/CentOS-Base.repo -O /etc/yum.repos.d/epel.repo 配置国内kubernetes源

cat < /etc/yum.repos.d/kubernetes.repo[kubernetes]name=Kubernetesbaseurl=配置docker源

wget -O /etc/yum.repos.d/docker-ce.repoyum clean all && yum makecache

注：如果是rathat系统将以上所有yum源里边的$releasever替换成7然后再执行yum clean all && yum makecache

2.7 三台master节点做ssh互信

[root@k8s-master03 ~]# cat fenfa.sh #!/bin/bashrm /root/.ssh/id_dsassh-keygen -t dsa -f /root/.ssh/id_dsa -N ""for ip in 92 93 94 dosshpass -p 123456 ssh-copy-id -i /root/.ssh/id_dsa.pub root@10.131.100.${ip} -o StrictHostKeyChecking=nodone

三、安装docker并配置

3.1 升级内核

执行uname -a #查看内核版本，低于3.10.0-514版本需要升级内核

yum install -y kernel #yum直接进行内核升级

3.2 安装docker18版本和其他相关组件

yum -y install docker-ce-18.06.3*

3.3 Docker 更换 存储方式 Docker切换OverLay(2)——提高性能，加快速度

官方地址：​​/u03 #如果 ftype=0 表示不是xfsumount /u03 mkfs.xfs -f /dev/mapper/rhel-u03 # 如果提示设备或资源忙 则执行fuser -km /u03mount /dev/mapper/rhel-u03 /u03 xfs_info /u03/ |grep ftype #出现ftype=1 就算完成

创建/etc/docker/daemon.json文件，并配置

cat >/etc/docker/daemon.json<

3） 重启docker并加入到开机自启动

systemctl restart docker && systemctl enable dockerdocker status docker

四、安装etcd集群

4.1 下载二进制包并安装并安装etcd（三个master节点都执行）

etcd下载地址：​​/u01/etcdtar zxf etcd-v3.3.10-linux-amd64.tar.gz -C /u01/rm -f /usr/bin/etcd /usr/bin/etcdctlln -s /u01/etcd-v3.3.10-linux-amd64/etcd /usr/bin/etcdln -s /u01/etcd-v3.3.10-linux-amd64/etcdctl /usr/bin/etcdctlmkdir /etc/etcd/etcdssl -p 2>/dev/null

4.2 生成etcd配置文件和系统启动文件（三个master节点都执行）

[root@k8s-master01 ~]# cat etcd.sh local_ip=`/sbin/ifconfig | grep 'inet'| grep -v '127.0.0.1' |grep -v "172.1*" | grep -v inet6 |cut -d: -f2 | awk '{ print $2}'|head -1`etcd01="10.131.100.92"etcd02="10.131.100.93"etcd03="10.131.100.94"cat > /etc/etcd/etcd.conf <

sh etcd.sh

4.3 配置etcd证书文件生成证书和私钥（在master01上操作）

1）配置证书生成文件

cd /etc/etcd/etcdsslcat > ca-config.json <

cat > ca-csr.json <

cat > etcd-csr.json <

4.4 下载cfssl 证书生成工具并生成证书然后拷贝至另外两个节点

cd /u01/wget +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64\cp -rp cfssl_linux-amd64 /usr/local/bin/cfssl\cp -rp cfssljson_linux-amd64 /usr/local/bin/cfssljson\cp -rp cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

cd /etc/etcd/etcdssl##生成 CA 证书和私钥cfssl gencert -initca ca-csr.json | cfssljson -bare ca###生成 etcd证书和私钥cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=etcd etcd-csr.json | cfssljson -bare etcd[root@k8s-master01 etcdssl]# ll总用量 36-rw-r--r-- 1 root root 290 12月 3 09:15 ca-config.json-rw-r--r-- 1 root root 997 12月 3 09:23 ca.csr-rw-r--r-- 1 root root 245 12月 3 09:15 ca-csr.json-rw------- 1 root root 1679 12月 3 09:23 ca-key.pem-rw-r--r-- 1 root root 1350 12月 3 09:23 ca.pem-rw-r--r-- 1 root root 1058 12月 3 09:23 etcd.csr-rw-r--r-- 1 root root 345 12月 3 09:23 etcd-csr.json-rw------- 1 root root 1679 12月 3 09:23 etcd-key.pem-rw-r--r-- 1 root root 1428 12月 3 09:23 etcd.pem###拷贝到其他服务器scp -r /etc/etcd/etcdssl 10.131.100.93:/etc/etcd/scp -r /etc/etcd/etcdssl 10.131.100.94:/etc/etcd/

4.5 加入开机启动并启动所有节点的etcd并验证集群

systemctl enable etcd && systemctl restart etcd[root@k8s-master01 etcdssl]# etcdctl --ca-file=/etc/etcd/etcdssl/ca.pem --cert-file=/etc/etcd/etcdssl/etcd.pem --key-file=/etc/etcd/etcdssl/etcd-key.pem cluster-healthmember 3b08f83c3b351284 is healthy: got healthy result from b276b92973c348d8 is healthy: got healthy result from ee911b7a6ae0c43a is healthy: got healthy result from is healthy

提示：如果有报错可以通过journalctl -xefu etcd 或 tailf /var/log/messages进行排查

五、Haproxy_docekr部署（做负载均衡）

5.1 配置ipvs 做内部负载均衡（三台master都执行）

cat > /etc/sysconfig/modules/ipvs.modules <

5.2 生成ha配置文件（三台master都执行）

cat >/etc/haproxy/haproxy.cfg<

5.3 启动ha容器（三台master都执行）

docker run -d --name my-haproxy \-v /etc/haproxy:/usr/local/etc/haproxy:ro \-p 8443:8443 \-p 1080:1080 \--restart always \192.168.220.84/kubernetes/haproxy:1.7.8-alpine

提示：部署完成后可通过 ​​admin 进行验证

六、部署keepalived_docker（做高可用）

6.1 直接启动keepalibed容器（三台master都执行）

docker run --net=host --cap-add=NET_ADMIN \-e KEEPALIVED_INTERFACE=eth0 \-e KEEPALIVED_VIRTUAL_IPS="#PYTHON2BASH:['10.0.0.253']" \ #vip地址-e KEEPALIVED_CHECK_PORT=8443 \-e KEEPALIVED_UNICAST_PEERS="#PYTHON2BASH:['10.131.100.92','10.131.100.93','10.131.100.94']" \ #master集群的地址-e KEEPALIVED_PASSWORD=admin \--name k8s-keepalived \--restart always \-d 192.168.220.84/kubernetes/osixia/keepalived:1.4.4

七、初始化集群

1） 安装其他相关组件

yum install kubelet-1.14.3-0 kubeadm-1.14.3-0 kubectl-1.14.3-0 kubernetes-cni-0.7.5

提示：kubectl为aipserver的客户端，nodes节点可不用安装

2） 编辑初始化配置文件

[root@k8s-master01 k8s]# cat config.yaml ---apiVersion: kubeproxy.config.k8s.io/v1alpha1kind: KubeProxyConfigurationfeatureGates: SupportIPVSProxyMode: truemode: ipvs---apiVersion: kubeadm.k8s.io/v1beta1bootstrapTokens:- groups: - system:bootstrappers:kubeadm:default-node-token token: n95wls.h1ifw0ln1mzlmfhu ttl: "0" usages: - signing - authenticationkind: InitConfigurationlocalAPIEndpoint: advertiseAddress: 10.131.100.92 #本地监听的地址 bindPort: 6443nodeRegistration: criSocket: /var/run/dockershim.sock name: k8s-master01 # kubect get nodes 查看的节点名称 taints: - effect: NoSchedule key: node-role.kubernetes.io/master---apiServer: timeoutForControlPlane: 4m0sapiVersion: kubeadm.k8s.io/v1beta1certificatesDir: /etc/kubernetes/pkiclusterName: kubernetescontrolPlaneEndpoint: "10.131.100.253:8443" #keepalived的vip地址controllerManager: {}dns: type: CoreDNSetcd: external: endpoints: - - - caFile: /etc/kubernetes/pki/etcd/ca.crt certFile: /etc/kubernetes/pki/apiserver-etcd-client.crt keyFile: /etc/kubernetes/pki/apiserver-etcd-client.key# local:# dataDir: /u03/etcd_dockerimageRepository: 192.168.220.84/kubernetes #harbor仓库的地址kind: ClusterConfigurationkubernetesVersion: v1.14.3networking: dnsDomain: cluster.local podSubnet: "10.244.0.0/16" # pod的网段 serviceSubnet: 10.96.0.0/12 # server的网段scheduler: {}

将etcd的证书文件拷贝到对应的目录下

cd /etc/etcd/etcdssl\cp -r ca.pem /etc/kubernetes/pki/etcd/ca.crt\cp -r etcd.pem /etc/kubernetes/pki/apiserver-etcd-client.crt\cp -r etcd-key.pem /etc/kubernetes/pki/apiserver-etcd-client.key

初始化集群

kubeadm init --config=/u01/k8s/config.yaml --experimental-upload-certs | tee /u01/kubeadm-init.log

提示：如果初始化不成功可用kubeadm reset 进行还原方便重新初始化

检查apiserver和etcd证书有效时间是否为100年

cd /etc/kubernetes/pki/ openssl x509 -in apiserver.crt -noout -text |grep ' Not ' Not Before: Dec 11 15:06:23 2019 GMT Not After : Nov 17 15:06:24 2119 GMT openssl x509 -in apiserver-etcd-client.crt -noout -text |grep ' Not ' Not Before: Dec 11 14:33:00 2019 GMT Not After : Nov 17 14:33:00 2119 GMT

配置kubelet访问apiservers

echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> ~/.bash_profilesource ~/.bash_profile

安装网络插件

kubectl apply -f calico.yaml

另外两台master节点加入集群(通过下边的kubeadm join 在另外两个master上运行，初始化成master节点)

[root@k8s-master01 ~]# grep -A 2 "kubeadm join" /u01/kubeadm-init.log |head -3 kubeadm join 10.0.0.253:8443 --token n95wls.h1ifw0ln1mzlmfhu \ --discovery-token-ca-cert-hash sha256:197b877bf7b05df875ee49e324d77ebdfac6977026cc6d58376af94112b450a6 \ --experimental-control-plane --certificate-key 258f15e292378bf6ac1d8a2a1de7c81d2a698e802624e3fac51b711cdfcc0f13

将node节点加入集群（通过一下kubeadm join 在node节点上执行，初始化成集群node节点）

[root@k8s-master01 ~]# grep -A 2 "kubeadm join" /u01/kubeadm-init.log | tail -2kubeadm join 10.0.0.253:8443 --token n95wls.h1ifw0ln1mzlmfhu \ --discovery-token-ca-cert-hash sha256:197b877bf7b05df875ee49e324d77ebdfac6977026cc6d58376af94112b450a6

验证集群状态（所有节点状态是ready表示集群安装成功）

[root@k8s-master01 ~]# kubectl get nodesNAME STATUS ROLES AGE VERSIONk8s-master01 Ready master 158m v1.14.3k8s-master02 Ready master 121m v1.14.3k8s-master03 Ready master 117m v1.14.3k8s-node01 Ready 113m v1.14.3

八、 遇到的问题

8.1 忘记加入机器的证书

kubeadm token create --print-join-command #生成加入集群的证书kubeadm init phase upload-certs --upload-certs #生成加入到master的新的证书#进行拼接 kubeadm join apiserver.com:6443 --token n3oixu.n0tz4q9k0p3qgbpf \ --discovery-token-ca-cert-hash sha256:a412994b2bb087e9970ebe7d83c45ef81b6b0e30c72e2a8a94e174bdd4b23b82 #加入集群的证书\ --control-plane --certificate-key a288aff17ed60a2541febb74f3cdc662f0f64922273cda2ee127f42b48ec912c #master证书

​

版权声明：本文内容由网络用户投稿，版权归原作者所有，本站不拥有其著作权，亦不承担相应法律责任。如果您发现本站中有涉嫌抄袭或描述失实的内容，请联系我们jiasou666@gmail.com 处理，核实后本网站将在24小时内删除侵权内容。