使用kubeasz进行Kubernetes自动化集群部署
Kubeasz架构
- 项目地址
https://github.com/easzlab/kubeasz
https://github.com/easzlab/kubeasz/blob/master/docs/setup/00-planning_and_overall_intro.md- 项目架构图及部署方式
每个Node节点上部署了轻量级Nginx作为代理指向Control节点，Kubelet和Kube-proxy中原本指向apiserver的地址替换为代理地址，由代理转为向COntrol报告存活及指令转发
部署Ansible并同步密钥到配置节点
#apt安装git和ansible
[root@K8s-ansible ~]#hostname
K8s-ansible.mooreyxia.com
[root@K8s-ansible ~]#apt update; apt install git ansible
#部署节点做python软连接
[root@K8s-ansible ~]#ln -s /usr/bin/python3 /usr/bin/python
#多主机密钥互通脚本
[root@K8s-ansible ~]#vim sshpass-copy.sh
[root@K8s-ansible ~]#cat sshpass-copy.sh
#!/bin/bash
#当前用户密码
PASS=123
#设置网段最小和最大的地址的尾数
BEGIN=211
END=219
IP=`hostname -I|awk '{print $1}'`
#IP=`ip a s eth0 | awk -F'[ /]+' 'NR==3{print $3}'`
NET=${IP%.*}.
. /etc/os-release
color () {
RES_COL=60
MOVE_TO_COL="echo -en \033[${RES_COL}G"
SETCOLOR_SUCCESS="echo -en \033[1;32m"
SETCOLOR_FAILURE="echo -en \033[1;31m"
SETCOLOR_WARNING="echo -en \033[1;33m"
SETCOLOR_NORMAL="echo -en E[0m"
echo -n "$1" && $MOVE_TO_COL
echo -n "["
if [ $2 = "success" -o $2 = "0" ] ;then
${SETCOLOR_SUCCESS}
echo -n $" OK "
elif [ $2 = "failure" -o $2 = "1" ] ;then
${SETCOLOR_FAILURE}
echo -n $"FAILED"
else
${SETCOLOR_WARNING}
echo -n $"WARNING"
fi
${SETCOLOR_NORMAL}
echo -n "]"
echo
}
#安装sshpass
install_sshpass() {
if [[ $ID =~ centos|rocky|rhel ]];then
rpm -q sshpass &> /dev/null || yum -y install sshpass
else
dpkg -l|grep -q sshpass || { sudo apt update;sudo apt -y install sshpass; }
fi
if [ $? -ne 0 ];then
color '安装 sshpass 失败!' 1
exit 1
fi
}
scan_host() {
[ -e ./SCANIP.log ] && rm -f SCANIP.log
for((i=$BEGIN;i /dev/null && echo "${NET}$i" >> SCANIP.log &
done
wait
}
push_ssh_key() {
#生成ssh key
[ -e ~/.ssh/id_rsa ] || ssh-keygen -P "" -f ~/.ssh/id_rsa
sshpass -p $PASS ssh-copy-id -o StrictHostKeyChecking=no ${USER}@$IP &>/dev/null
ip_list=(`sort -t . -k 4 -n SCANIP.log`)
for ip in ${ip_list[*]};do
sshpass -p $PASS scp -o StrictHostKeyChecking=no -r ~/.ssh ${USER}@${ip}: &>/dev/null
done
#把.ssh/known_hosts拷贝到所有主机，使它们第一次互相访问时不需要输入yes回车
for ip in ${ip_list[*]};do
scp ~/.ssh/known_hosts ${USER}@${ip}:.ssh/ &>/dev/null
color "$ip" 0
done
#创建python软连接，Ansible程序需要使用
for ip in ${ip_list[*]};do
ssh ${ip} ln -sv /usr/bin/python3 /usr/bin/python
echo "${ip} /usr/bin/python3 软连接创建完成"
done
}
install_sshpass
scan_host
push_ssh_key
[root@K8s-ansible ~]#bash sshpass-copy.sh
#测试直连
[root@K8s-ansible ~]#
[root@K8s-ansible ~]#ssh 192.168.11.219
Welcome to Ubuntu 22.04.1 LTS (GNU/Linux 5.15.0-60-generic x86_64)
* Documentation: https://help.ubuntu.com
* Management: https://landscape.canonical.com
* Support: https://ubuntu.com/advantage
This system has been minimized by removing packages and content that are
not required on a system that users do not log into.
To restore this content, you can run the 'unminimize' command.
Last login: Fri Mar 24 03:22:29 2023 from 192.168.11.3
[root@K8s-etcd03 ~]#exit
logout
Connection to 192.168.11.219 closed.配置Kubeasz项目和组件
#下载工具脚本ezdown
[root@K8s-ansible ~]#export release=3.3.4
[root@K8s-ansible ~]#wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
--2023-03-23 10:26:01-- https://github.com/easzlab/kubeasz/releases/download/3.5.0/ezdown
Resolving github.com (github.com)... 20.205.243.166
Connecting to github.com (github.com)|20.205.243.166|:443... connected.
HTTP request sent, awaiting response... 302 Found
Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/110401202/9992736a-b85c-4701-977c-188c9bace190?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20230323%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230323T102602Z&X-Amz-Expires=300&X-Amz-Signature=ab0660ef01bd7893b92106acc444bce5d022325bee3257c9486cb8659642809f&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=110401202&response-content-dispositinotallow=attachment%3B%20filename%3Dezdown&response-content-type=application%2Foctet-stream [following]
--2023-03-23 10:26:02-- https://objects.githubusercontent.com/github-production-release-asset-2e65be/110401202/9992736a-b85c-4701-977c-188c9bace190?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20230323%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230323T102602Z&X-Amz-Expires=300&X-Amz-Signature=ab0660ef01bd7893b92106acc444bce5d022325bee3257c9486cb8659642809f&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=110401202&response-content-dispositinotallow=attachment%3B%20filename%3Dezdown&response-content-type=application%2Foctet-stream
Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.109.133, 185.199.108.133, 185.199.111.133, ...
Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.109.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 25417 (25K) [application/octet-stream]
Saving to: ‘ezdown’
ezdown 100%[============================================================================================>] 24.82K --.-KB/s in 0.1s
2023-03-23 10:26:04 (206 KB/s) - ‘ezdown’ saved [25417/25417]
[root@K8s-ansible ~]#chmod +x ./ezdown
#根据需求调整安装的docker版本与Kubernetes版本
#脚本中的镜像是提前放置在了kubeasz官方镜像中，所以会先安装Docker，下载并运行官方镜像，再从官方镜像中复制到本地，之后会删除官方镜像
[root@K8s-ansible ~]#vim ezdown
...
# default settings, can be overridden by cmd line options, see usage
DOCKER_VER=20.10.18
KUBEASZ_VER=3.5.0
K8S_BIN_VER=v1.26.0
EXT_BIN_VER=1.6.3
SYS_PKG_VER=0.5.2
HARBOR_VER=v2.1.5
REGISTRY_MIRROR=CN
# images downloaded by default(with '-D')
calicoVer=v3.23.5
dnsNodeCacheVer=1.22.13
corednsVer=1.9.3
dashboardVer=v2.7.0
dashboardMetricsScraperVer=v1.0.8
metricsVer=v0.5.2
pauseVer=3.9
# images not downloaded by default(only download with '-X')
ciliumVer=1.12.4
flannelVer=v0.19.2
nfsProvisinotallow=v4.0.2
promChartVer=39.11.0
# images not downloaded
kubeRouterVer=v0.3.1
kubeOvnVer=v1.5.3
...
#下载kubeasz代码、二进制、默认容器镜像（更多关于ezdown的参数，运行./ezdown 查看）
# 国内环境
./ezdown -D
# 海外环境
#./ezdown -D -m standard
#时间会比较长，耐心等待
[root@K8s-ansible ~]#./ezdown -D
#下载完成后查看镜像
[root@K8s-ansible ~]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
082653541eab registry:2 "/entrypoint.sh /etc…" 21 minutes ago Up 21 minutes local_registry
[root@K8s-ansible ~]#docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
registry 2 0d153fadf70b 5 weeks ago 24.2MB
easzlab/kubeasz 3.5.0 42c6ecf49faf 3 months ago 182MB
easzlab/kubeasz-k8s-bin v1.26.0 76f7eb409903 3 months ago 1.17GB
easzlab/kubeasz-ext-bin 1.6.3 ff684f2f91f2 3 months ago 543MB
easzlab.io.local:5000/calico/kube-controllers v3.23.5 ea5536b1fa4a 4 months ago 127MB
calico/kube-controllers v3.23.5 ea5536b1fa4a 4 months ago 127MB
calico/cni v3.23.5 1c979d623de9 4 months ago 254MB
easzlab.io.local:5000/calico/cni v3.23.5 1c979d623de9 4 months ago 254MB
calico/node v3.23.5 b6e6ee0788f2 4 months ago 207MB
easzlab.io.local:5000/calico/node v3.23.5 b6e6ee0788f2 4 months ago 207MB
easzlab/pause 3.9 78d53e70b442 5 months ago 744kB
easzlab.io.local:5000/easzlab/pause 3.9 78d53e70b442 5 months ago 744kB
easzlab/k8s-dns-node-cache 1.22.13 7b3b529c5a5a 5 months ago 64.3MB
easzlab.io.local:5000/easzlab/k8s-dns-node-cache 1.22.13 7b3b529c5a5a 5 months ago 64.3MB
kubernetesui/dashboard v2.7.0 07655ddf2eeb 6 months ago 246MB
easzlab.io.local:5000/kubernetesui/dashboard v2.7.0 07655ddf2eeb 6 months ago 246MB
kubernetesui/metrics-scraper v1.0.8 115053965e86 9 months ago 43.8MB
easzlab.io.local:5000/kubernetesui/metrics-scraper v1.0.8 115053965e86 9 months ago 43.8MB
coredns/coredns 1.9.3 5185b96f0bec 9 months ago 48.8MB
easzlab.io.local:5000/coredns/coredns 1.9.3 5185b96f0bec 9 months ago 48.8MB
easzlab/metrics-server v0.5.2 f965999d664b 16 months ago 64.3MB
easzlab.io.local:5000/easzlab/metrics-server v0.5.2 f965999d664b 16 months ago 64.3MB
#生成了一些配置文件用来管理Kubernetes集群
[root@K8s-ansible ~]#cd /etc/kubeasz/
[root@K8s-ansible kubeasz]#ll
total 136
drwxrwxr-x 12 root root 4096 Mar 23 12:07 ./
drwxr-xr-x 87 root root 4096 Mar 23 12:07 ../
drwxrwxr-x 3 root root 4096 Dec 20 14:09 .github/
-rw-rw-r-- 1 root root 301 Dec 20 13:32 .gitignore
-rw-rw-r-- 1 root root 5609 Dec 20 13:32 README.md
-rw-rw-r-- 1 root root 20304 Dec 20 13:32 ansible.cfg
drwxr-xr-x 3 root root 4096 Mar 23 12:07 bin/
drwxrwxr-x 8 root root 4096 Dec 20 14:09 docs/
drwxr-xr-x 3 root root 4096 Mar 23 12:20 down/
drwxrwxr-x 2 root root 4096 Dec 20 14:09 example/
-rwxrwxr-x 1 root root 26123 Dec 20 13:32 ezctl*
-rwxrwxr-x 1 root root 25417 Dec 20 13:32 ezdown*
drwxrwxr-x 10 root root 4096 Dec 20 14:09 manifests/
drwxrwxr-x 2 root root 4096 Dec 20 14:09 pics/
drwxrwxr-x 2 root root 4096 Dec 20 14:09 playbooks/
drwxrwxr-x 22 root root 4096 Dec 20 14:09 roles/
drwxrwxr-x 2 root root 4096 Dec 20 14:09 tools/
[root@K8s-ansible kubeasz]#cat ezctl
#!/bin/bash
# Create & manage k8s clusters
set -o nounset
set -o errexit
#set -o xtrace
function usage() {
echo -e " 33[33mUsage: 33[0m ezctl COMMAND [args]"
cat to switch default kubeconfig of the cluster
new to start a new k8s deploy with name 'cluster'
setup to setup a cluster, also supporting a step-by-step way
start to start all of the k8s services stopped by 'ezctl stop'
stop to stop all of the k8s services temporarily
upgrade to upgrade the k8s cluster
destroy to destroy the k8s cluster
backup to backup the cluster state (etcd snapshot)
restore to restore the cluster state from backups
start-aio to quickly setup an all-in-one cluster with default settings
Cluster ops:
add-etcd to add a etcd-node to the etcd cluster
add-master to add a master node to the k8s cluster
add-node to add a work node to the k8s cluster
del-etcd to delete a etcd-node from the etcd cluster
del-master to delete a master node from the k8s cluster
del-node to delete a work node from the k8s cluster
Extra operation:
kca-renew to force renew CA certs and all the other certs (with caution)
kcfg-adm to manage client kubeconfig of the k8s cluster
Use "ezctl help " for more information about a given command.
EOF
}
function logger() {
TIMESTAMP=$(date +'%Y-%m-%d %H:%M:%S')
case "$1" in
debug)
echo -e "$TIMESTAMP 33[36mDEBUG 33[0m $2"
;;
info)
echo -e "$TIMESTAMP 33[32mINFO 33[0m $2"
;;
warn)
echo -e "$TIMESTAMP 33[33mWARN 33[0m $2"
;;
error)
echo -e "$TIMESTAMP 33[31mERROR 33[0m $2"
;;
*)
;;
esac
}
function help-info() {
case "$1" in
(setup)
usage-setup
;;
(add-etcd)
echo -e "read more > 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-etcd.md'"
;;
(add-master)
echo -e "read more > 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-master.md'"
;;
(add-node)
echo -e "read more > 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-node.md'"
;;
(del-etcd)
echo -e "read more > 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-etcd.md'"
;;
(del-master)
echo -e "read more > 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-master.md'"
;;
(del-node)
echo -e "read more > 'https://github.com/easzlab/kubeasz/blob/master/docs/op/op-node.md'"
;;
(kca-renew)
echo -e "WARNNING: this command should be used with caution"
echo -e "force to recreate CA certs and all of the others certs used in the cluster"
echo -e "it should be used only when the admin.conf leaked"
;;
(kcfg-adm)
usage-kcfg-adm
;;
(*)
echo -e "todo: help info $1"
;;
esac
}
function usage-kcfg-adm(){
echo -e " 33[33mUsage: 33[0m ezctl kcfg-adm "
cat :
-A to add a client kubeconfig with a newly created user
-D to delete a client kubeconfig with the existed user
-L to list all of the users
-e to set expiry of the user certs in hours (ex. 24h, 8h, 240h)
-t to set a user-type (admin or view)
-u to set a user-name prefix
examples: ./ezctl kcfg-adm test-k8s -L
./ezctl kcfg-adm default -A -e 240h -t admin -u jack
./ezctl kcfg-adm default -D -u jim-202101162141
EOF
}
function usage-setup(){
echo -e " 33[33mUsage: 33[0m ezctl setup "
cat 将镜像上传到自建Harbor
#为了保证镜像的安全性和使用便捷，将镜像上传到自建Harbor
#确认要上传的镜像
[root@K8s-ansible ~]#docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
registry 2 0d153fadf70b 5 weeks ago 24.2MB
easzlab/kubeasz 3.5.0 42c6ecf49faf 3 months ago 182MB
easzlab/kubeasz-k8s-bin v1.26.0 76f7eb409903 3 months ago 1.17GB
easzlab/kubeasz-ext-bin 1.6.3 ff684f2f91f2 3 months ago 543MB
calico/kube-controllers v3.23.5 ea5536b1fa4a 4 months ago 127MB
easzlab.io.local:5000/calico/kube-controllers v3.23.5 ea5536b1fa4a 4 months ago 127MB
calico/cni v3.23.5 1c979d623de9 4 months ago 254MB
easzlab.io.local:5000/calico/cni v3.23.5 1c979d623de9 4 months ago 254MB
calico/node v3.23.5 b6e6ee0788f2 4 months ago 207MB
easzlab.io.local:5000/calico/node v3.23.5 b6e6ee0788f2 4 months ago 207MB
easzlab/pause 3.9 78d53e70b442 5 months ago 744kB
easzlab.io.local:5000/easzlab/pause 3.9 78d53e70b442 5 months ago 744kB
easzlab/k8s-dns-node-cache 1.22.13 7b3b529c5a5a 5 months ago 64.3MB
easzlab.io.local:5000/easzlab/k8s-dns-node-cache 1.22.13 7b3b529c5a5a 5 months ago 64.3MB
kubernetesui/dashboard v2.7.0 07655ddf2eeb 6 months ago 246MB
easzlab.io.local:5000/kubernetesui/dashboard v2.7.0 07655ddf2eeb 6 months ago 246MB
kubernetesui/metrics-scraper v1.0.8 115053965e86 9 months ago 43.8MB
easzlab.io.local:5000/kubernetesui/metrics-scraper v1.0.8 115053965e86 9 months ago 43.8MB
coredns/coredns 1.9.3 5185b96f0bec 10 months ago 48.8MB
easzlab.io.local:5000/coredns/coredns 1.9.3 5185b96f0bec 10 months ago 48.8MB
easzlab/metrics-server v0.5.2 f965999d664b 16 months ago 64.3MB
easzlab.io.local:5000/easzlab/metrics-server v0.5.2 f965999d664b 16 months ago 64.3MB
#建立一个证书目录
[root@K8s-ansible ~]#mkdir -pv /etc/docker/certs.d/K8s-harbor01.mooreyxia.com/
mkdir: created directory '/etc/docker/certs.d'
mkdir: created directory '/etc/docker/certs.d/K8s-harbor01.mooreyxia.com/'
#证书复制到客户端
[root@K8s-harbor01 ~]#cd /data/harbor/certs/
[root@K8s-harbor01 certs]#pwd
/data/harbor/certs
[root@K8s-harbor01 certs]#ll
total 36
drwxr-xr-x 2 root root 4096 Mar 18 08:02 ./
drwxr-xr-x 9 root root 4096 Mar 18 04:56 ../
-rw-r--r-- 1 root root 2195 Mar 18 08:02 K8s-harbor01.mooreyxia.com.cert
-rw-r--r-- 1 root root 2195 Mar 18 04:59 K8s-harbor01.mooreyxia.com.crt
-rw-r--r-- 1 root root 1724 Mar 18 04:58 K8s-harbor01.mooreyxia.com.csr
-rw------- 1 root root 3272 Mar 18 04:57 K8s-harbor01.mooreyxia.com.key
-rw-r--r-- 1 root root 2049 Mar 18 04:57 ca.crt
-rw------- 1 root root 3272 Mar 18 04:57 ca.key
-rw-r--r-- 1 root root 288 Mar 18 04:58 v3.ext
[root@K8s-harbor01 certs]#scp K8s-harbor01.mooreyxia.com.key 192.168.11.205:/etc/docker/certs.d/K8s-harbor01.mooreyxia.com/
The authenticity of host '192.168.11.205 (192.168.11.205)' can't be established.
ED25519 key fingerprint is SHA256:11syElL/FfYd7XcJqX+HZPUZcIoBgpIVER+1YBY3Cl8.
This host key is known by the following other names/addresses:
~/.ssh/known_hosts:1: [hashed name]
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '192.168.11.205' (ED25519) to the list of known hosts.
root@192.168.11.205's password:
K8s-harbor01.mooreyxia.com.key 100% 3272 480.9KB/s 00:00
[root@K8s-harbor01 certs]#scp K8s-harbor01.mooreyxia.com.cert 192.168.11.205:/etc/docker/certs.d/K8s-harbor01.mooreyxia.com/
root@192.168.11.205's password:
K8s-harbor01.mooreyxia.com.cert 100% 2195 317.5KB/s 00:00
[root@K8s-harbor01 certs]#scp ca.crt 192.168.11.205:/etc/docker/certs.d/K8s-harbor01.mooreyxia.com/
root@192.168.11.205's password:
ca.crt 100% 2049 196.8KB/s 00:00
#登录harbor
[root@K8s-ansible ~]#docker login K8s-harbor01.mooreyxia.com
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
#将镜像打标签后上传到自建Harbor
[root@K8s-ansible ~]#docker tag easzlab/kubeasz:3.5.0 K8s-harbor01.mooreyxia.com/kubernetes/easzlab/kubeasz:3.5.0
[root@K8s-ansible ~]#docker push K8s-harbor01.mooreyxia.com/kubernetes/easzlab/kubeasz:3.5.0
The push refers to repository [K8s-harbor01.mooreyxia.com/kubernetes/easzlab/kubeasz]
a17947b74426: Pushed
10178cfed86d: Pushed
89e2f983ddba: Pushed
3777ff7cada1: Pushed
d3ffccd4dc39: Pushed
bdd2dbc0f630: Pushed
994393dc58e7: Pushed
3.5.0: digest: sha256:ef9e91d5c1214717f11717bca1744715b9df684103415456999fe187220f073a size: 1791
#脚本批量上传
[root@K8s-ansible ~]#cat docker_push.sh
!#/bin/bash
harborIP="K8s-harbor01.mooreyxia.com"
folder="kubernetes"
images="
easzlab/kubeasz-ext-bin:1.6.3
calico/kube-controllers:v3.23.5
calico/cni:v3.23.5
calico/node:v3.23.5
easzlab/pause:3.9
easzlab/k8s-dns-node-cache:1.22.13
kubernetesui/dashboard:v2.7.0
kubernetesui/metrics-scraper:v1.0.8
coredns/coredns:1.9.3
easzlab/metrics-server:v0.5.2
"
for image in ${images};do
#标记
docker tag ${image} $harborIP/$folder/${image}
#上传
docker push $harborIP/$folder/${image}
done
生成集群配置文件(hosts和config.yml)
[root@K8s-ansible kubeasz]#pwd
/etc/kubeasz
[root@K8s-ansible kubeasz]#./ezctl --help
Usage: ezctl COMMAND [args]
-------------------------------------------------------------------------------------
Cluster setups:
list to list all of the managed clusters
checkout to switch default kubeconfig of the cluster
new to start a new k8s deploy with name 'cluster'
setup to setup a cluster, also supporting a step-by-step way
start to start all of the k8s services stopped by 'ezctl stop'
stop to stop all of the k8s services temporarily
upgrade to upgrade the k8s cluster
destroy to destroy the k8s cluster
backup to backup the cluster state (etcd snapshot)
restore to restore the cluster state from backups
start-aio to quickly setup an all-in-one cluster with default settings
Cluster ops:
add-etcd to add a etcd-node to the etcd cluster
add-master to add a master node to the k8s cluster
add-node to add a work node to the k8s cluster
del-etcd to delete a etcd-node from the etcd cluster
del-master to delete a master node from the k8s cluster
del-node to delete a work node from the k8s cluster
Extra operation:
kca-renew to force renew CA certs and all the other certs (with caution)
kcfg-adm to manage client kubeconfig of the k8s cluster
Use "ezctl help " for more information about a given command.
#生成集群环境hosts配置文件和组件config配置文件
#可以根据需求定义多个集群环境k8s-cluster1、k8s-cluster2、k8s-cluster3、.......
[root@K8s-ansible kubeasz]#./ezctl new k8s-cluster1
2023-03-23 12:50:30 DEBUG generate custom cluster files in /etc/kubeasz/clusters/k8s-cluster1
2023-03-23 12:50:30 DEBUG set versions
2023-03-23 12:50:30 DEBUG cluster k8s-cluster1: files successfully created.
2023-03-23 12:50:30 INFO next steps 1: to config '/etc/kubeasz/clusters/k8s-cluster1/hosts'
2023-03-23 12:50:30 INFO next steps 2: to config '/etc/kubeasz/clusters/k8s-cluster1/config.yml' 修改host集群配置文件
[root@K8s-ansible kubeasz]#vim /etc/kubeasz/clusters/k8s-cluster1/hosts
[root@K8s-ansible kubeasz]#cat /etc/kubeasz/clusters/k8s-cluster1/hosts
# 'etcd' cluster should have odd member(s) (1,3,5,...)
[etcd]
192.168.11.217
192.168.11.218
192.168.11.219
# master node(s)
[kube_master]
192.168.11.211
192.168.11.212
# work node(s)
[kube_node]
192.168.11.214
192.168.11.215
# [optional] harbor server, a private docker registry
# 'NEW_INSTALL': 'true' to install a harbor server; 'false' to integrate with existed one
[harbor]
#192.168.11.8 NEW_INSTALL=false
# [optional] loadbalance for accessing k8s from outside
[ex_lb]
#192.168.11.6 LB_ROLE=backup EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443
#192.168.11.7 LB_ROLE=master EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443
# [optional] ntp server for the cluster
[chrony]
#192.168.11.1
[all:vars]
# --------- Main Variables ---------------
# Secure port for apiservers
SECURE_PORT="6443"
# Cluster container-runtime supported: docker, containerd
# if k8s version >= 1.24, docker is not supported
CONTAINER_RUNTIME="containerd"
#网络组件
# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
CLUSTER_NETWORK="calico"
#proxy规则基于哪种安全模式
# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"
#Service网段
# K8S Service CIDR, not overlap with node(host) networking
SERVICE_CIDR="10.100.0.0/16"
#Pod网段
# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
CLUSTER_CIDR="10.200.0.0/16"
#k授权8s使用的端口
# NodePort Range
NODE_PORT_RANGE="30000-32767"
#集群域名后缀
# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="mooreyxia.local"
# -------- Additional Variables (don't change the default value right now) ---
#二进制同步位置，方便直连主机后快捷执行
# Binaries Directory
bin_dir="/usr/local/bin"
#部署目录
# Deploy Directory (kubeasz workspace)
base_dir="/etc/kubeasz"
# Directory for a specific cluster
cluster_dir="{{ base_dir }}/clusters/k8s-cluster1"
# CA and other components cert/key Directory
ca_dir="/etc/kubernetes/ssl"修改config服务配置文件
#下面配置文件中的插件自动安装都关闭，稍后会手动安装
[root@K8s-ansible ~]#vim /etc/kubeasz/clusters/k8s-cluster1/config.yml
[root@K8s-ansible ~]#cat /etc/kubeasz/clusters/k8s-cluster1/config.yml
############################
# prepare
############################
# 可选离线安装系统软件包 (offline|online)
INSTALL_SOURCE: "online"
# 可选进行系统安全加固 github.com/dev-sec/ansible-collection-hardening
OS_HARDEN: false
############################
# role:deploy
############################
# default: ca will expire in 100 years
# default: certs issued by the ca will expire in 50 years
CA_EXPIRY: "876000h"
CERT_EXPIRY: "438000h"
# force to recreate CA and other certs, not suggested to set 'true'
CHANGE_CA: false
# kubeconfig 配置参数
CLUSTER_NAME: "cluster1"
CONTEXT_NAME: "context-{{ CLUSTER_NAME }}"
# k8s version
K8S_VER: "1.26.0"
############################
# role:etcd
############################
# 设置不同的wal目录，可以避免磁盘io竞争，提高性能
ETCD_DATA_DIR: "/var/lib/etcd"
ETCD_WAL_DIR: ""
############################
# role:runtime [containerd,docker] 二选一
############################
# ------------------------------------------- containerd
# [.]启用容器仓库镜像
ENABLE_MIRROR_REGISTRY: true
# [containerd]基础容器镜像
#SANDBOX_IMAGE: "easzlab.io.local:5000/easzlab/pause:3.9"
SANDBOX_IMAGE: "K8s-harbor01.mooreyxia.com/kubernetes/easzlab/pausei:3.9"
# [containerd]容器持久化存储目录
CONTAINERD_STORAGE_DIR: "/var/lib/containerd"
# ------------------------------------------- docker
# [docker]容器存储目录
DOCKER_STORAGE_DIR: "/var/lib/docker"
# [docker]开启Restful API
ENABLE_REMOTE_API: false
# [docker]信任的HTTP仓库
INSECURE_REG: '["http://easzlab.io.local:5000"]'
############################
# role:kube-master
############################
# k8s 集群 master 节点证书配置，可以添加多个ip和域名（比如增加公网ip和域名）
MASTER_CERT_HOSTS:
- "192.168.11.241"
- "k8s.mooreyxia.net"
#- "www.test.com"
# node 节点上 pod 网段掩码长度（决定每个节点最多能分配的pod ip地址）
# 如果flannel 使用 --kube-subnet-mgr 参数，那么它将读取该设置为每个节点分配pod网段
# https://github.com/coreos/flannel/issues/847
NODE_CIDR_LEN: 24
############################
# role:kube-node
############################
# Kubelet 根目录
KUBELET_ROOT_DIR: "/var/lib/kubelet"
# node节点最大pod 数
#MAX_PODS: 110
MAX_PODS: 500
# 配置为kube组件（kubelet,kube-proxy,dockerd等）预留的资源量
# 数值设置详见templates/kubelet-config.yaml.j2
KUBE_RESERVED_ENABLED: "no"
# k8s 官方不建议草率开启 system-reserved, 除非你基于长期监控，了解系统的资源占用状况；
# 并且随着系统运行时间，需要适当增加资源预留，数值设置详见templates/kubelet-config.yaml.j2
# 系统预留设置基于 4c/8g 虚机，最小化安装系统服务，如果使用高性能物理机可以适当增加预留
# 另外，集群安装时候apiserver等资源占用会短时较大，建议至少预留1g内存
SYS_RESERVED_ENABLED: "no"
############################
# role:network [flannel,calico,cilium,kube-ovn,kube-router]
############################
# ------------------------------------------- flannel
# [flannel]设置flannel 后端"host-gw","vxlan"等
FLANNEL_BACKEND: "vxlan"
DIRECT_ROUTING: false
# [flannel]
flannel_ver: "v0.19.2"
# ------------------------------------------- calico
# [calico] IPIP隧道模式可选项有: [Always, CrossSubnet, Never],跨子网可以配置为Always与CrossSubnet(公有云建议使用always比较省事，其他的话需要修改各自公有云的网络配置，具体可以参考各个公有云说明)
# 其次CrossSubnet为隧道+BGP路由混合模式可以提升网络性能，同子网配置为Never即可.
CALICO_IPV4POOL_IPIP: "Always"
# [calico]设置 calico-node使用的host IP，bgp邻居通过该地址建立，可手工指定也可以自动发现
IP_AUTODETECTION_METHOD: "can-reach={{ groups['kube_master'][0] }}"
# [calico]设置calico 网络 backend: brid, vxlan, none
CALICO_NETWORKING_BACKEND: "brid"
# [calico]设置calico 是否使用route reflectors
# 如果集群规模超过50个节点，建议启用该特性
CALICO_RR_ENABLED: false
# CALICO_RR_NODES 配置route reflectors的节点，如果未设置默认使用集群master节点
# CALICO_RR_NODES: ["192.168.1.1", "192.168.1.2"]
CALICO_RR_NODES: []
# [calico]更新支持calico 版本: ["3.19", "3.23"]
calico_ver: "v3.23.5"
# [calico]calico 主版本
calico_ver_main: "{{ calico_ver.split('.')[0] }}.{{ calico_ver.split('.')[1] }}"
# ------------------------------------------- cilium
# [cilium]镜像版本
cilium_ver: "1.12.4"
cilium_connectivity_check: true
cilium_hubble_enabled: false
cilium_hubble_ui_enabled: false
# ------------------------------------------- kube-ovn
# [kube-ovn]选择 OVN DB and OVN Control Plane 节点，默认为第一个master节点
OVN_DB_NODE: "{{ groups['kube_master'][0] }}"
# [kube-ovn]离线镜像tar包
kube_ovn_ver: "v1.5.3"
# ------------------------------------------- kube-router
# [kube-router]公有云上存在限制，一般需要始终开启 ipinip；自有环境可以设置为 "subnet"
OVERLAY_TYPE: "full"
# [kube-router]NetworkPolicy 支持开关
FIREWALL_ENABLE: true
# [kube-router]kube-router 镜像版本
kube_router_ver: "v0.3.1"
busybox_ver: "1.28.4"
############################
# role:cluster-addon
############################
# coredns 自动安装
#dns_install: "yes"
dns_install: "no"
corednsVer: "1.9.3"
#ENABLE_LOCAL_DNS_CACHE: true
ENABLE_LOCAL_DNS_CACHE: false
dnsNodeCacheVer: "1.22.13"
# 设置 local dns cache 地址
LOCAL_DNS_CACHE: "169.254.20.10"
# metric server 自动安装
#metricsserver_install: "yes"
metricsserver_install: "no"
metricsVer: "v0.5.2"
# dashboard 自动安装
#dashboard_install: "yes"
dashboard_install: "no"
dashboardVer: "v2.7.0"
dashboardMetricsScraperVer: "v1.0.8"
# prometheus 自动安装
prom_install: "no"
prom_namespace: "monitor"
prom_chart_ver: "39.11.0"
# nfs-provisioner 自动安装
nfs_provisioner_install: "no"
nfs_provisioner_namespace: "kube-system"
nfs_provisioner_ver: "v4.0.2"
nfs_storage_class: "managed-nfs-storage"
nfs_server: "192.168.1.10"
nfs_path: "/data/nfs"
# network-check 自动安装
network_check_enabled: false
network_check_schedule: "*/5 * * * *"
############################
# role:harbor
############################
# harbor version，完整版本号
HARBOR_VER: "v2.1.5"
HARBOR_DOMAIN: "harbor.easzlab.io.local"
HARBOR_PATH: /var/data
HARBOR_TLS_PORT: 8443
HARBOR_REGISTRY: "{{ HARBOR_DOMAIN }}:{{ HARBOR_TLS_PORT }}"
# if set 'false', you need to put certs named harbor.pem and harbor-key.pem in directory 'down'
HARBOR_SELF_SIGNED_CERT: true
# install extra component
HARBOR_WITH_NOTARY: false
HARBOR_WITH_TRIVY: false
HARBOR_WITH_CLAIR: false
HARBOR_WITH_CHARTMUSEUM: true执行kubeasz-setup01环境初始化脚本
------------------------------------------------------------------
#Ansible执行的环境配置脚本主要是以下内容，先确保1-6环境配置成功，其余按需要配置即可
PLAY_BOOK="dummy.yml"
case "$2" in
(01|prepare)
PLAY_BOOK="01.prepare.yml"
;;
(02|etcd)
PLAY_BOOK="02.etcd.yml"
;;
(03|container-runtime)
PLAY_BOOK="03.runtime.yml"
;;
(04|kube-master)
PLAY_BOOK="04.kube-master.yml"
;;
(05|kube-node)
PLAY_BOOK="05.kube-node.yml"
;;
(06|network)
PLAY_BOOK="06.network.yml"
;;
(07|cluster-addon)
PLAY_BOOK="07.cluster-addon.yml"
;;
(90|all)
PLAY_BOOK="90.setup.yml"
;;
(10|ex-lb)
PLAY_BOOK="10.ex-lb.yml"
;;
(11|harbor)
PLAY_BOOK="11.harbor.yml"
;;
(*)
usage-setup
exit 1
;;
esac
------------------------------------------------------------------
[root@K8s-ansible ~]#cd /etc/kubeasz/
[root@K8s-ansible kubeasz]#ls
README.md ansible.cfg bin clusters docs down example ezctl ezdown manifests pics playbooks roles tools
#自动化执行的所有内容都在当前目录下，比如环境初始化，包括软件环境预装与卸载、环境参数优化等等
[root@K8s-ansible kubeasz]#tree roles/prepare/
roles/prepare/
├── files
│ └── sctp.conf
├── tasks
│ ├── centos.yml
│ ├── common.yml
│ ├── main.yml
│ ├── offline.yml
│ └── ubuntu.yml
└── templates
├── 10-k8s-modules.conf.j2
├── 30-k8s-ulimits.conf.j2
├── 95-k8s-journald.conf.j2
└── 95-k8s-sysctl.conf.j2
3 directories, 10 files
#执行初始化步骤
[root@K8s-ansible kubeasz]#./ezctl setup k8s-cluster1 01
ansible-playbook -i clusters/k8s-cluster1/hosts -e @clusters/k8s-cluster1/config.yml playbooks/01.prepare.yml
2023-03-24 06:51:14 INFO cluster:k8s-cluster1 setup step:01 begins in 5s, press any key to abort:
PLAY [kube_master,kube_node,etcd] ************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [192.168.11.214]
ok: [192.168.11.212]
ok: [192.168.11.215]
ok: [192.168.11.217]
ok: [192.168.11.211]
ok: [192.168.11.218]
ok: [192.168.11.219]
PLAY [localhost] *****************************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [localhost]
TASK [deploy : prepare some dirs] ************************************************************************************************************************************************
ok: [localhost] => (item=/etc/kubeasz/clusters/k8s-cluster1/ssl)
ok: [localhost] => (item=/etc/kubeasz/clusters/k8s-cluster1/backup)
ok: [localhost] => (item=/etc/kubeasz/clusters/k8s-cluster1/yml)
ok: [localhost] => (item=~/.kube)
TASK [deploy : 本地设置 bin 目录权限] ****************************************************************************************************************************************************
ok: [localhost]
TASK [deploy : 读取ca证书stat信息] *****************************************************************************************************************************************************
ok: [localhost]
TASK [deploy : 准备kubectl使用的admin证书签名请求] ******************************************************************************************************************************************
ok: [localhost]
TASK [deploy : 创建admin证书与私钥] *****************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] ***********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置客户端认证参数] ********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] **********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] **********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 安装kubeconfig] *****************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kube-proxy 证书签名请求] **********************************************************************************************************************************************
ok: [localhost]
TASK [deploy : 创建 kube-proxy证书与私钥] ***********************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] ***********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置客户端认证参数] ********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] **********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] **********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kube-controller-manager 证书签名请求] *********************************************************************************************************************************
ok: [localhost]
TASK [deploy : 创建 kube-controller-manager证书与私钥] **********************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] ***********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置认证参数] ***********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] **********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] **********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 准备kube-scheduler 证书签名请求] ******************************************************************************************************************************************
ok: [localhost]
TASK [deploy : 创建 kube-scheduler证书与私钥] *******************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置集群参数] ***********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置认证参数] ***********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 设置上下文参数] **********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 选择默认上下文] **********************************************************************************************************************************************************
changed: [localhost]
TASK [deploy : 本地创建 ezdown/ezctl 工具的软连接] *****************************************************************************************************************************************
ok: [localhost] => (item=ezdown)
ok: [localhost] => (item=ezctl)
TASK [deploy : ansible 控制端创建 kubectl 软链接] ****************************************************************************************************************************************
ok: [localhost]
PLAY [kube_master,kube_node,etcd] ************************************************************************************************************************************************
TASK [prepare : apt更新缓存刷新] *******************************************************************************************************************************************************
ok: [192.168.11.215]
ok: [192.168.11.214]
ok: [192.168.11.217]
ok: [192.168.11.211]
ok: [192.168.11.212]
ok: [192.168.11.218]
ok: [192.168.11.219]
TASK [prepare : 删除ubuntu默认安装] ****************************************************************************************************************************************************
changed: [192.168.11.215] => (item=ufw)
changed: [192.168.11.214] => (item=ufw)
changed: [192.168.11.211] => (item=ufw)
changed: [192.168.11.212] => (item=ufw)
changed: [192.168.11.217] => (item=ufw)
changed: [192.168.11.215] => (item=lxd)
changed: [192.168.11.214] => (item=lxd)
changed: [192.168.11.211] => (item=lxd)
changed: [192.168.11.212] => (item=lxd)
changed: [192.168.11.217] => (item=lxd)
changed: [192.168.11.215] => (item=lxcfs)
changed: [192.168.11.214] => (item=lxcfs)
changed: [192.168.11.211] => (item=lxcfs)
changed: [192.168.11.217] => (item=lxcfs)
changed: [192.168.11.212] => (item=lxcfs)
changed: [192.168.11.215] => (item=lxc-common)
changed: [192.168.11.214] => (item=lxc-common)
changed: [192.168.11.211] => (item=lxc-common)
changed: [192.168.11.217] => (item=lxc-common)
changed: [192.168.11.212] => (item=lxc-common)
changed: [192.168.11.218] => (item=ufw)
changed: [192.168.11.219] => (item=ufw)
changed: [192.168.11.218] => (item=lxd)
changed: [192.168.11.219] => (item=lxd)
changed: [192.168.11.218] => (item=lxcfs)
changed: [192.168.11.219] => (item=lxcfs)
changed: [192.168.11.218] => (item=lxc-common)
changed: [192.168.11.219] => (item=lxc-common)
TASK [prepare : 安装 ubuntu/debian基础软件] ********************************************************************************************************************************************
ok: [192.168.11.215]
ok: [192.168.11.211]
ok: [192.168.11.212]
ok: [192.168.11.214]
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 准备 journal 日志相关目录] ***********************************************************************************************************************************************
changed: [192.168.11.211] => (item=/etc/systemd/journald.conf.d)
changed: [192.168.11.212] => (item=/etc/systemd/journald.conf.d)
changed: [192.168.11.215] => (item=/etc/systemd/journald.conf.d)
changed: [192.168.11.214] => (item=/etc/systemd/journald.conf.d)
changed: [192.168.11.217] => (item=/etc/systemd/journald.conf.d)
ok: [192.168.11.212] => (item=/var/log/journal)
ok: [192.168.11.211] => (item=/var/log/journal)
ok: [192.168.11.214] => (item=/var/log/journal)
ok: [192.168.11.215] => (item=/var/log/journal)
ok: [192.168.11.217] => (item=/var/log/journal)
changed: [192.168.11.218] => (item=/etc/systemd/journald.conf.d)
changed: [192.168.11.219] => (item=/etc/systemd/journald.conf.d)
ok: [192.168.11.218] => (item=/var/log/journal)
ok: [192.168.11.219] => (item=/var/log/journal)
TASK [prepare : 优化设置 journal 日志] *************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.214]
changed: [192.168.11.212]
changed: [192.168.11.215]
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 重启 journald 服务] **************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
changed: [192.168.11.211]
changed: [192.168.11.217]
changed: [192.168.11.212]
changed: [192.168.11.219]
changed: [192.168.11.218]
TASK [prepare : 禁用系统 swap] *******************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.217]
changed: [192.168.11.212]
changed: [192.168.11.211]
changed: [192.168.11.215]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 删除fstab swap 相关配置] ***********************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
changed: [192.168.11.212]
changed: [192.168.11.211]
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 转换内核版本为浮点数] ******************************************************************************************************************************************************
ok: [192.168.11.211]
ok: [192.168.11.212]
ok: [192.168.11.214]
ok: [192.168.11.215]
ok: [192.168.11.217]
ok: [192.168.11.218]
ok: [192.168.11.219]
TASK [prepare : 加载内核模块] **********************************************************************************************************************************************************
ok: [192.168.11.211] => (item=br_netfilter)
ok: [192.168.11.214] => (item=br_netfilter)
ok: [192.168.11.212] => (item=br_netfilter)
ok: [192.168.11.215] => (item=br_netfilter)
changed: [192.168.11.217] => (item=br_netfilter)
changed: [192.168.11.211] => (item=ip_vs)
changed: [192.168.11.214] => (item=ip_vs)
changed: [192.168.11.212] => (item=ip_vs)
changed: [192.168.11.217] => (item=ip_vs)
changed: [192.168.11.215] => (item=ip_vs)
changed: [192.168.11.214] => (item=ip_vs_rr)
changed: [192.168.11.211] => (item=ip_vs_rr)
changed: [192.168.11.212] => (item=ip_vs_rr)
changed: [192.168.11.217] => (item=ip_vs_rr)
changed: [192.168.11.215] => (item=ip_vs_rr)
changed: [192.168.11.214] => (item=ip_vs_wrr)
changed: [192.168.11.211] => (item=ip_vs_wrr)
changed: [192.168.11.212] => (item=ip_vs_wrr)
changed: [192.168.11.217] => (item=ip_vs_wrr)
changed: [192.168.11.215] => (item=ip_vs_wrr)
changed: [192.168.11.214] => (item=ip_vs_sh)
changed: [192.168.11.211] => (item=ip_vs_sh)
changed: [192.168.11.212] => (item=ip_vs_sh)
changed: [192.168.11.217] => (item=ip_vs_sh)
changed: [192.168.11.215] => (item=ip_vs_sh)
ok: [192.168.11.214] => (item=nf_conntrack)
ok: [192.168.11.211] => (item=nf_conntrack)
ok: [192.168.11.212] => (item=nf_conntrack)
ok: [192.168.11.217] => (item=nf_conntrack)
ok: [192.168.11.215] => (item=nf_conntrack)
changed: [192.168.11.219] => (item=br_netfilter)
changed: [192.168.11.218] => (item=br_netfilter)
changed: [192.168.11.219] => (item=ip_vs)
changed: [192.168.11.218] => (item=ip_vs)
changed: [192.168.11.219] => (item=ip_vs_rr)
changed: [192.168.11.218] => (item=ip_vs_rr)
changed: [192.168.11.219] => (item=ip_vs_wrr)
changed: [192.168.11.218] => (item=ip_vs_wrr)
changed: [192.168.11.219] => (item=ip_vs_sh)
changed: [192.168.11.218] => (item=ip_vs_sh)
ok: [192.168.11.219] => (item=nf_conntrack)
ok: [192.168.11.218] => (item=nf_conntrack)
TASK [prepare : 尝试加载nf_conntrack_ipv4] *******************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.214]
changed: [192.168.11.215]
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 启用systemd自动加载模块服务] ***********************************************************************************************************************************************
ok: [192.168.11.211]
ok: [192.168.11.214]
ok: [192.168.11.215]
ok: [192.168.11.212]
ok: [192.168.11.217]
ok: [192.168.11.218]
ok: [192.168.11.219]
TASK [prepare : 增加内核模块开机加载配置] ****************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.214]
changed: [192.168.11.215]
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 设置系统参数] **********************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.215]
changed: [192.168.11.212]
changed: [192.168.11.214]
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 生效系统参数] **********************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.214]
changed: [192.168.11.215]
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 创建 systemd 配置目录] *************************************************************************************************************************************************
changed: [192.168.11.212]
changed: [192.168.11.211]
changed: [192.168.11.217]
changed: [192.168.11.214]
changed: [192.168.11.215]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 设置系统 ulimits] ****************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.215]
changed: [192.168.11.214]
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 把SCTP列入内核模块黑名单] **************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
changed: [192.168.11.217]
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : prepare some dirs] ***********************************************************************************************************************************************
ok: [192.168.11.211] => (item=/usr/local/bin)
ok: [192.168.11.212] => (item=/usr/local/bin)
ok: [192.168.11.214] => (item=/usr/local/bin)
ok: [192.168.11.215] => (item=/usr/local/bin)
ok: [192.168.11.217] => (item=/usr/local/bin)
changed: [192.168.11.211] => (item=/etc/kubernetes/ssl)
changed: [192.168.11.214] => (item=/etc/kubernetes/ssl)
changed: [192.168.11.212] => (item=/etc/kubernetes/ssl)
changed: [192.168.11.215] => (item=/etc/kubernetes/ssl)
changed: [192.168.11.217] => (item=/etc/kubernetes/ssl)
changed: [192.168.11.214] => (item=/root/.kube)
changed: [192.168.11.211] => (item=/root/.kube)
changed: [192.168.11.215] => (item=/root/.kube)
changed: [192.168.11.212] => (item=/root/.kube)
changed: [192.168.11.217] => (item=/root/.kube)
changed: [192.168.11.214] => (item=/etc/cni/net.d)
changed: [192.168.11.211] => (item=/etc/cni/net.d)
changed: [192.168.11.215] => (item=/etc/cni/net.d)
changed: [192.168.11.212] => (item=/etc/cni/net.d)
changed: [192.168.11.217] => (item=/etc/cni/net.d)
ok: [192.168.11.218] => (item=/usr/local/bin)
ok: [192.168.11.219] => (item=/usr/local/bin)
changed: [192.168.11.218] => (item=/etc/kubernetes/ssl)
changed: [192.168.11.219] => (item=/etc/kubernetes/ssl)
changed: [192.168.11.218] => (item=/root/.kube)
changed: [192.168.11.219] => (item=/root/.kube)
changed: [192.168.11.218] => (item=/etc/cni/net.d)
changed: [192.168.11.219] => (item=/etc/cni/net.d)
TASK [prepare : symlink /usr/bin/python -> /usr/bin/python3] *********************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.214]
changed: [192.168.11.215]
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 写入环境变量$PATH] *****************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.215]
changed: [192.168.11.214]
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [prepare : 添加 local registry hosts 解析] **************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.214]
changed: [192.168.11.215]
changed: [192.168.11.217]
changed: [192.168.11.219]
changed: [192.168.11.218]
PLAY RECAP ***********************************************************************************************************************************************************************
192.168.11.211 : ok=23 changed=18 unreachable=0 failed=0 skipped=97 rescued=0 ignored=0
192.168.11.212 : ok=23 changed=18 unreachable=0 failed=0 skipped=97 rescued=0 ignored=0
192.168.11.214 : ok=23 changed=18 unreachable=0 failed=0 skipped=97 rescued=0 ignored=0
192.168.11.215 : ok=23 changed=18 unreachable=0 failed=0 skipped=97 rescued=0 ignored=0
192.168.11.217 : ok=23 changed=19 unreachable=0 failed=0 skipped=97 rescued=0 ignored=0
192.168.11.218 : ok=23 changed=19 unreachable=0 failed=0 skipped=97 rescued=0 ignored=0
192.168.11.219 : ok=23 changed=19 unreachable=0 failed=0 skipped=97 rescued=0 ignored=0
localhost : ok=31 changed=21 unreachable=0 failed=0 skipped=13 rescued=0 ignored=0
[root@K8s-ansible kubeasz]#执行kubeasz-setup02-etcd集群脚本
[root@K8s-ansible kubeasz]#tree roles/etcd/
roles/etcd/
├── clean-etcd.yml
├── defaults
│ └── main.yml
├── tasks
│ └── main.yml
└── templates
├── etcd-csr.json.j2
└── etcd.service.j2
3 directories, 5 files
[root@K8s-ansible kubeasz]#./ezctl setup k8s-cluster1 02
ansible-playbook -i clusters/k8s-cluster1/hosts -e @clusters/k8s-cluster1/config.yml playbooks/02.etcd.yml
2023-03-26 05:25:59 INFO cluster:k8s-cluster1 setup step:02 begins in 5s, press any key to abort:
PLAY [etcd] **********************************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [192.168.11.217]
ok: [192.168.11.219]
ok: [192.168.11.218]
TASK [etcd : prepare some dirs] **************************************************************************************************************************************************
ok: [192.168.11.219]
ok: [192.168.11.217]
ok: [192.168.11.218]
TASK [etcd : 下载etcd二进制文件] ********************************************************************************************************************************************************
ok: [192.168.11.218] => (item=etcd)
ok: [192.168.11.219] => (item=etcd)
ok: [192.168.11.217] => (item=etcd)
ok: [192.168.11.218] => (item=etcdctl)
ok: [192.168.11.219] => (item=etcdctl)
ok: [192.168.11.217] => (item=etcdctl)
TASK [etcd : 创建etcd证书请求] *********************************************************************************************************************************************************
changed: [192.168.11.217]
TASK [etcd : 创建 etcd证书和私钥] *******************************************************************************************************************************************************
changed: [192.168.11.217]
TASK [etcd : 分发etcd证书相关] *********************************************************************************************************************************************************
changed: [192.168.11.217] => (item=ca.pem)
changed: [192.168.11.219] => (item=ca.pem)
changed: [192.168.11.218] => (item=ca.pem)
changed: [192.168.11.217] => (item=etcd.pem)
changed: [192.168.11.219] => (item=etcd.pem)
changed: [192.168.11.218] => (item=etcd.pem)
changed: [192.168.11.217] => (item=etcd-key.pem)
changed: [192.168.11.219] => (item=etcd-key.pem)
changed: [192.168.11.218] => (item=etcd-key.pem)
TASK [etcd : 创建etcd的systemd unit文件] **********************************************************************************************************************************************
changed: [192.168.11.218]
changed: [192.168.11.217]
changed: [192.168.11.219]
TASK [etcd : 开机启用etcd服务] *********************************************************************************************************************************************************
changed: [192.168.11.217]
changed: [192.168.11.218]
changed: [192.168.11.219]
TASK [etcd : 开启etcd服务] ***********************************************************************************************************************************************************
changed: [192.168.11.217]
changed: [192.168.11.219]
changed: [192.168.11.218]
TASK [etcd : 以轮询的方式等待服务同步完成] *****************************************************************************************************************************************************
changed: [192.168.11.217]
changed: [192.168.11.219]
changed: [192.168.11.218]
PLAY RECAP ***********************************************************************************************************************************************************************
192.168.11.217 : ok=10 changed=7 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
192.168.11.218 : ok=8 changed=5 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
192.168.11.219 : ok=8 changed=5 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
#在任意etcd节点上调用集群心跳检测
[root@K8s-etcd01 ~]#cat check_etcdcluster.sh
#!/bin/bash
IP="
192.168.11.217
192.168.11.218
192.168.11.219
"
for ip in ${IP}; do
ETCDCTL_API=3
/usr/local/bin/etcdctl
--endpoints=https://${ip}:2379
--cacert=/etc/kubernetes/ssl/ca.pem
--cert=/etc/kubernetes/ssl/etcd.pem
--key=/etc/kubernetes/ssl/etcd-key.pem endpoint health;
done
[root@K8s-etcd01 ~]#bash check_etcdcluster.sh
https://192.168.11.217:2379 is healthy: successfully committed proposal: took = 34.363076ms
https://192.168.11.218:2379 is healthy: successfully committed proposal: took = 30.262915ms
https://192.168.11.219:2379 is healthy: successfully committed proposal: took = 41.485995ms执行kubeasz-setup03-部署运行时脚本
[root@K8s-ansible kubeasz]#tree roles/containerd/
roles/containerd/
├── tasks
│ └── main.yml
└── templates
├── config.toml.j2
├── containerd.service.j2
└── crictl.yaml.j2
#修改部署运行时文件中的下载地址为私有harbor
[root@K8s-ansible kubeasz]#cat roles/containerd/templates/config.toml.j2
...
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."nvcr.io"]
endpoint = ["https://ngc.nju.edu.cn"]
{% endif %}
------------------------指定镜像仓库-----------------------------------
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."K8s-harbor01.mooreyxia.com"]
endpoint = ["https://k8s-harbor01.mooreyxia.com"]
[plugins."io.containerd.grpc.v1.cri".registry.configs."K8s-harbor01.mooreyxia.com".tls]
insecure_skip_verify = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."K8s-harbor01.mooreyxia.com".auth]
username = "admin"
password = "123456"
-----------------------------------------------------------
[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[root@K8s-ansible kubeasz]#./ezctl setup k8s-cluster1 03
ansible-playbook -i clusters/k8s-cluster1/hosts -e @clusters/k8s-cluster1/config.yml playbooks/03.runtime.yml
2023-03-26 12:37:01 INFO cluster:k8s-cluster1 setup step:03 begins in 5s, press any key to abort:
PLAY [kube_master,kube_node] *****************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [192.168.11.211]
ok: [192.168.11.212]
ok: [192.168.11.215]
ok: [192.168.11.214]
TASK [containerd : 获取是否已经安装containerd] *******************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.214]
changed: [192.168.11.212]
changed: [192.168.11.215]
TASK [containerd : 准备containerd相关目录] *********************************************************************************************************************************************
ok: [192.168.11.215] => (item=/usr/local/bin)
ok: [192.168.11.214] => (item=/usr/local/bin)
ok: [192.168.11.212] => (item=/usr/local/bin)
ok: [192.168.11.211] => (item=/usr/local/bin)
changed: [192.168.11.215] => (item=/etc/containerd)
changed: [192.168.11.211] => (item=/etc/containerd)
changed: [192.168.11.214] => (item=/etc/containerd)
changed: [192.168.11.212] => (item=/etc/containerd)
TASK [containerd : 加载内核模块 overlay] ***********************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.212]
changed: [192.168.11.211]
changed: [192.168.11.215]
TASK [containerd : 下载 containerd 二进制文件] ******************************************************************************************************************************************
changed: [192.168.11.212] => (item=containerd)
changed: [192.168.11.211] => (item=containerd)
changed: [192.168.11.215] => (item=containerd)
changed: [192.168.11.214] => (item=containerd)
changed: [192.168.11.212] => (item=containerd-shim)
changed: [192.168.11.211] => (item=containerd-shim)
changed: [192.168.11.214] => (item=containerd-shim)
changed: [192.168.11.215] => (item=containerd-shim)
changed: [192.168.11.212] => (item=containerd-shim-runc-v1)
changed: [192.168.11.211] => (item=containerd-shim-runc-v1)
changed: [192.168.11.214] => (item=containerd-shim-runc-v1)
changed: [192.168.11.215] => (item=containerd-shim-runc-v1)
changed: [192.168.11.212] => (item=containerd-shim-runc-v2)
changed: [192.168.11.211] => (item=containerd-shim-runc-v2)
changed: [192.168.11.214] => (item=containerd-shim-runc-v2)
changed: [192.168.11.215] => (item=containerd-shim-runc-v2)
changed: [192.168.11.212] => (item=crictl)
changed: [192.168.11.215] => (item=crictl)
changed: [192.168.11.211] => (item=crictl)
changed: [192.168.11.214] => (item=crictl)
changed: [192.168.11.212] => (item=ctr)
changed: [192.168.11.215] => (item=ctr)
changed: [192.168.11.214] => (item=ctr)
changed: [192.168.11.212] => (item=runc)
changed: [192.168.11.211] => (item=ctr)
changed: [192.168.11.215] => (item=runc)
changed: [192.168.11.214] => (item=runc)
changed: [192.168.11.211] => (item=runc)
TASK [containerd : 添加 crictl 自动补全] ***********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.214]
changed: [192.168.11.212]
changed: [192.168.11.215]
TASK [containerd : 创建 containerd 配置文件] *******************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.215]
TASK [containerd : 创建systemd unit文件] *********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.215]
changed: [192.168.11.214]
changed: [192.168.11.212]
TASK [containerd : 创建 crictl 配置] *************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [containerd : 开机启用 containerd 服务] *******************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [containerd : 开启 containerd 服务] *********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.215]
changed: [192.168.11.212]
changed: [192.168.11.214]
TASK [containerd : 轮询等待containerd服务运行] *******************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.214]
changed: [192.168.11.212]
changed: [192.168.11.215]
PLAY RECAP ***********************************************************************************************************************************************************************
192.168.11.211 : ok=12 changed=11 unreachable=0 failed=0 skipped=18 rescued=0 ignored=0
192.168.11.212 : ok=12 changed=11 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0
192.168.11.214 : ok=12 changed=11 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0
192.168.11.215 : ok=12 changed=11 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0
#在Worker和Node节点都已经部署Containerd
[root@K8s-master01 ~]#systemctl status containerd
● containerd.service - containerd container runtime
Loaded: loaded (/etc/systemd/system/containerd.service; enabled; vendor preset: enabled)
Active: active (running) since Sun 2023-03-26 12:37:46 UTC; 2min 52s ago
Docs: https://containerd.io
Process: 3548 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)
Main PID: 3549 (containerd)
Tasks: 8 (limit: 2237)
Memory: 12.8M
CPU: 173ms
CGroup: /system.slice/containerd.service
└─3549 /usr/local/bin/containerd
Mar 26 12:37:46 K8s-master01.mooreyxia.com containerd[3549]: time="2023-03-26T12:37:46.772804046Z" level=error msg="failed to load cni during init, please check CRI plugin status
before setting up network for pods" error="cni config load failed: no network config found in /etc/cni/net.d: cni plugin not initialized: failed to load cni config"
Mar 26 12:37:46 K8s-master01.mooreyxia.com containerd[3549]: time="2023-03-26T12:37:46.774093806Z" level=info msg="Start subscribing containerd event"
Mar 26 12:37:46 K8s-master01.mooreyxia.com containerd[3549]: time="2023-03-26T12:37:46.774828046Z" level=info msg=serving... address=/run/containerd/containerd.sock.ttrpc
Mar 26 12:37:46 K8s-master01.mooreyxia.com containerd[3549]: time="2023-03-26T12:37:46.775161686Z" level=info msg=serving... address=/run/containerd/containerd.sock
Mar 26 12:37:46 K8s-master01.mooreyxia.com containerd[3549]: time="2023-03-26T12:37:46.775442186Z" level=info msg="containerd successfully booted in 0.174434s"
Mar 26 12:37:46 K8s-master01.mooreyxia.com containerd[3549]: time="2023-03-26T12:37:46.774894357Z" level=info msg="Start recovering state"
Mar 26 12:37:46 K8s-master01.mooreyxia.com containerd[3549]: time="2023-03-26T12:37:46.779703985Z" level=info msg="Start event monitor"
Mar 26 12:37:46 K8s-master01.mooreyxia.com containerd[3549]: time="2023-03-26T12:37:46.779827068Z" level=info msg="Start snapshots syncer"
Mar 26 12:37:46 K8s-master01.mooreyxia.com containerd[3549]: time="2023-03-26T12:37:46.779892814Z" level=info msg="Start cni network conf syncer for default"
Mar 26 12:37:46 K8s-master01.mooreyxia.com containerd[3549]: time="2023-03-26T12:37:46.779928641Z" level=info msg="Start streaming server"
#测试containerd从私有harbor拉取镜像
[root@K8s-master01 ~]#crictl pull K8s-harbor01.mooreyxia.com/kubernetes/easzlab/pause:3.9
Image is up to date for sha256:78d53e70b442be4f222242eb4944faa14df80ab9536a9bc6f2131defd4bc872d执行kubeasz-setup04-kube-master集群脚本
[root@K8s-ansible kubeasz]#tree roles/kube-master/
roles/kube-master/
├── tasks
│ └── main.yml
├── templates
│ ├── aggregator-proxy-csr.json.j2
│ ├── kube-apiserver.service.j2
│ ├── kube-controller-manager.service.j2
│ ├── kube-scheduler.service.j2
│ └── kubernetes-csr.json.j2
└── vars
└── main.yml
3 directories, 7 files
[root@K8s-ansible kubeasz]#./ezctl setup k8s-cluster1 04
ansible-playbook -i clusters/k8s-cluster1/hosts -e @clusters/k8s-cluster1/config.yml playbooks/04.kube-master.yml
2023-03-26 13:20:27 INFO cluster:k8s-cluster1 setup step:04 begins in 5s, press any key to abort:
PLAY [kube_master] ***************************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [192.168.11.212]
ok: [192.168.11.211]
TASK [kube-lb : prepare some dirs] ***********************************************************************************************************************************************
changed: [192.168.11.212] => (item=/etc/kube-lb/sbin)
changed: [192.168.11.211] => (item=/etc/kube-lb/sbin)
changed: [192.168.11.211] => (item=/etc/kube-lb/logs)
changed: [192.168.11.212] => (item=/etc/kube-lb/logs)
changed: [192.168.11.211] => (item=/etc/kube-lb/conf)
changed: [192.168.11.212] => (item=/etc/kube-lb/conf)
TASK [kube-lb : 下载二进制文件kube-lb(nginx)] *******************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-lb : 创建kube-lb的配置文件] **************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-lb : 创建kube-lb的systemd unit文件] ****************************************************************************************************************************************
changed: [192.168.11.212]
changed: [192.168.11.211]
TASK [kube-lb : 开机启用kube-lb服务] ***************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-lb : 开启kube-lb服务] *****************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] *********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-master : 下载 kube_master 二进制] ******************************************************************************************************************************************
changed: [192.168.11.211] => (item=kube-apiserver)
changed: [192.168.11.212] => (item=kube-apiserver)
changed: [192.168.11.212] => (item=kube-controller-manager)
changed: [192.168.11.211] => (item=kube-controller-manager)
changed: [192.168.11.212] => (item=kube-scheduler)
changed: [192.168.11.211] => (item=kube-scheduler)
changed: [192.168.11.212] => (item=kubectl)
changed: [192.168.11.211] => (item=kubectl)
TASK [kube-master : 分发controller/scheduler kubeconfig配置文件] ***********************************************************************************************************************
changed: [192.168.11.211] => (item=kube-controller-manager.kubeconfig)
changed: [192.168.11.212] => (item=kube-controller-manager.kubeconfig)
changed: [192.168.11.211] => (item=kube-scheduler.kubeconfig)
changed: [192.168.11.212] => (item=kube-scheduler.kubeconfig)
TASK [kube-master : 创建 kubernetes 证书签名请求] ****************************************************************************************************************************************
changed: [192.168.11.212]
ok: [192.168.11.211]
TASK [kube-master : 创建 kubernetes 证书和私钥] *****************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-master : 创建 aggregator proxy证书签名请求] ***********************************************************************************************************************************
changed: [192.168.11.211]
ok: [192.168.11.212]
TASK [kube-master : 创建 aggregator-proxy证书和私钥] ************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-master : 分发 kubernetes证书] *********************************************************************************************************************************************
changed: [192.168.11.212] => (item=ca.pem)
changed: [192.168.11.211] => (item=ca.pem)
changed: [192.168.11.212] => (item=ca-key.pem)
changed: [192.168.11.211] => (item=ca-key.pem)
changed: [192.168.11.212] => (item=kubernetes.pem)
changed: [192.168.11.211] => (item=kubernetes.pem)
changed: [192.168.11.212] => (item=kubernetes-key.pem)
changed: [192.168.11.211] => (item=kubernetes-key.pem)
changed: [192.168.11.212] => (item=aggregator-proxy.pem)
changed: [192.168.11.211] => (item=aggregator-proxy.pem)
changed: [192.168.11.212] => (item=aggregator-proxy-key.pem)
changed: [192.168.11.211] => (item=aggregator-proxy-key.pem)
TASK [kube-master : 替换 kubeconfig 的 apiserver 地址] ********************************************************************************************************************************
changed: [192.168.11.212] => (item=/etc/kubernetes/kube-controller-manager.kubeconfig)
changed: [192.168.11.211] => (item=/etc/kubernetes/kube-controller-manager.kubeconfig)
changed: [192.168.11.211] => (item=/etc/kubernetes/kube-scheduler.kubeconfig)
changed: [192.168.11.212] => (item=/etc/kubernetes/kube-scheduler.kubeconfig)
TASK [kube-master : 创建 master 服务的 systemd unit 文件] *******************************************************************************************************************************
changed: [192.168.11.211] => (item=kube-apiserver.service)
changed: [192.168.11.212] => (item=kube-apiserver.service)
changed: [192.168.11.211] => (item=kube-controller-manager.service)
changed: [192.168.11.212] => (item=kube-controller-manager.service)
changed: [192.168.11.211] => (item=kube-scheduler.service)
changed: [192.168.11.212] => (item=kube-scheduler.service)
TASK [kube-master : enable master 服务] ********************************************************************************************************************************************
changed: [192.168.11.212]
changed: [192.168.11.211]
TASK [kube-master : 启动 master 服务] ************************************************************************************************************************************************
changed: [192.168.11.212]
changed: [192.168.11.211]
TASK [kube-master : 轮询等待kube-apiserver启动] ****************************************************************************************************************************************
changed: [192.168.11.212]
changed: [192.168.11.211]
TASK [kube-master : 轮询等待kube-controller-manager启动] *******************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-master : 轮询等待kube-scheduler启动] ****************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-master : 复制kubectl.kubeconfig] ****************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-master : 替换 kubeconfig 的 apiserver 地址] ********************************************************************************************************************************
ok: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-master : 轮询等待master服务启动完成] ********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-master : 获取user:kubernetes是否已经绑定对应角色] *********************************************************************************************************************************
changed: [192.168.11.211]
TASK [kube-master : 创建user:kubernetes角色绑定] ***************************************************************************************************************************************
changed: [192.168.11.211]
TASK [kube-node : 创建kube_node 相关目录] **********************************************************************************************************************************************
changed: [192.168.11.212] => (item=/var/lib/kubelet)
changed: [192.168.11.211] => (item=/var/lib/kubelet)
changed: [192.168.11.212] => (item=/var/lib/kube-proxy)
changed: [192.168.11.211] => (item=/var/lib/kube-proxy)
TASK [kube-node : 下载 kubelet,kube-proxy 二进制和基础 cni plugins] **********************************************************************************************************************
ok: [192.168.11.211] => (item=kubectl)
ok: [192.168.11.212] => (item=kubectl)
changed: [192.168.11.212] => (item=kubelet)
changed: [192.168.11.211] => (item=kubelet)
changed: [192.168.11.211] => (item=kube-proxy)
changed: [192.168.11.212] => (item=kube-proxy)
changed: [192.168.11.211] => (item=bridge)
changed: [192.168.11.212] => (item=bridge)
changed: [192.168.11.211] => (item=host-local)
changed: [192.168.11.212] => (item=host-local)
changed: [192.168.11.211] => (item=loopback)
changed: [192.168.11.212] => (item=loopback)
TASK [kube-node : 添加 kubectl 自动补全] ***********************************************************************************************************************************************
changed: [192.168.11.212]
changed: [192.168.11.211]
TASK [kube-node : 准备kubelet 证书签名请求] **********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 创建 kubelet 证书与私钥] **********************************************************************************************************************************************
changed: [192.168.11.212]
changed: [192.168.11.211]
TASK [kube-node : 设置集群参数] ********************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 设置客户端认证参数] *****************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 设置上下文参数] *******************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 选择默认上下文] *******************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 分发ca 证书] *******************************************************************************************************************************************************
ok: [192.168.11.212]
ok: [192.168.11.211]
TASK [kube-node : 分发kubelet 证书] **************************************************************************************************************************************************
changed: [192.168.11.212] => (item=kubelet.pem)
changed: [192.168.11.211] => (item=kubelet.pem)
changed: [192.168.11.212] => (item=kubelet-key.pem)
changed: [192.168.11.211] => (item=kubelet-key.pem)
TASK [kube-node : 分发kubeconfig] **************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 准备 cni配置文件] ****************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 创建kubelet的配置文件] ************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 创建kubelet的systemd unit文件] **************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 开机启用kubelet 服务] ************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 开启kubelet 服务] **************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 分发 kube-proxy.kubeconfig配置文件] **********************************************************************************************************************************
changed: [192.168.11.212]
changed: [192.168.11.211]
TASK [kube-node : 替换 kube-proxy.kubeconfig 的 apiserver 地址] ***********************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 创建kube-proxy 配置] ***********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 创建kube-proxy 服务文件] *********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 开机启用kube-proxy 服务] *********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 开启kube-proxy 服务] ***********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 轮询等待kube-proxy启动] **********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : 轮询等待kubelet启动] *************************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
FAILED - RETRYING: 轮询等待node达到Ready状态 (8 retries left).
TASK [kube-node : 轮询等待node达到Ready状态] *********************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : Setting worker role name] **************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : Setting master role name] **************************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-node : Making master nodes SchedulingDisabled] ************************************************************************************************************************
changed: [192.168.11.211]
changed: [192.168.11.212]
PLAY RECAP ***********************************************************************************************************************************************************************
192.168.11.211 : ok=56 changed=52 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0
192.168.11.212 : ok=54 changed=51 unreachable=0 failed=0 skipped=1 rescued=0 ignored=0
#确认是否可以使用kubectl
[root@K8s-ansible kubeasz]#kubectl get node
NAME STATUS ROLES AGE VERSION
192.168.11.211 Ready,SchedulingDisabled master 16m v1.24.10
192.168.11.212 Ready,SchedulingDisabled master 16m v1.24.10执行kubeasz-setup05-kube-node集群脚本
[root@K8s-ansible kubeasz]#tree roles/kube-node/
roles/kube-node/
├── tasks
│ ├── create-kubelet-kubeconfig.yml
│ └── main.yml
├── templates
│ ├── cni-default.conf.j2
│ ├── kube-proxy-config.yaml.j2
│ ├── kube-proxy.service.j2
│ ├── kubelet-config.yaml.j2
│ ├── kubelet-csr.json.j2
│ └── kubelet.service.j2
└── vars
└── main.yml
3 directories, 9 files
[root@K8s-ansible kubeasz]#./ezctl setup k8s-cluster1 05
ansible-playbook -i clusters/k8s-cluster1/hosts -e @clusters/k8s-cluster1/config.yml playbooks/05.kube-node.yml
2023-03-26 13:45:44 INFO cluster:k8s-cluster1 setup step:05 begins in 5s, press any key to abort:
PLAY [kube_node] *****************************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [192.168.11.215]
ok: [192.168.11.214]
TASK [kube-lb : prepare some dirs] ***********************************************************************************************************************************************
changed: [192.168.11.215] => (item=/etc/kube-lb/sbin)
changed: [192.168.11.214] => (item=/etc/kube-lb/sbin)
changed: [192.168.11.214] => (item=/etc/kube-lb/logs)
changed: [192.168.11.215] => (item=/etc/kube-lb/logs)
changed: [192.168.11.214] => (item=/etc/kube-lb/conf)
changed: [192.168.11.215] => (item=/etc/kube-lb/conf)
TASK [kube-lb : 下载二进制文件kube-lb(nginx)] *******************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-lb : 创建kube-lb的配置文件] **************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-lb : 创建kube-lb的systemd unit文件] ****************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-lb : 开机启用kube-lb服务] ***************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-lb : 开启kube-lb服务] *****************************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] *********************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 创建kube_node 相关目录] **********************************************************************************************************************************************
changed: [192.168.11.214] => (item=/var/lib/kubelet)
changed: [192.168.11.215] => (item=/var/lib/kubelet)
changed: [192.168.11.214] => (item=/var/lib/kube-proxy)
changed: [192.168.11.215] => (item=/var/lib/kube-proxy)
TASK [kube-node : 下载 kubelet,kube-proxy 二进制和基础 cni plugins] **********************************************************************************************************************
changed: [192.168.11.214] => (item=kubectl)
changed: [192.168.11.215] => (item=kubectl)
changed: [192.168.11.214] => (item=kubelet)
changed: [192.168.11.215] => (item=kubelet)
changed: [192.168.11.214] => (item=kube-proxy)
changed: [192.168.11.215] => (item=kube-proxy)
changed: [192.168.11.214] => (item=bridge)
changed: [192.168.11.215] => (item=bridge)
changed: [192.168.11.214] => (item=host-local)
changed: [192.168.11.215] => (item=host-local)
changed: [192.168.11.214] => (item=loopback)
changed: [192.168.11.215] => (item=loopback)
TASK [kube-node : 添加 kubectl 自动补全] ***********************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-node : 准备kubelet 证书签名请求] **********************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 创建 kubelet 证书与私钥] **********************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 设置集群参数] ********************************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-node : 设置客户端认证参数] *****************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 设置上下文参数] *******************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 选择默认上下文] *******************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 分发ca 证书] *******************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 分发kubelet 证书] **************************************************************************************************************************************************
changed: [192.168.11.214] => (item=kubelet.pem)
changed: [192.168.11.215] => (item=kubelet.pem)
changed: [192.168.11.214] => (item=kubelet-key.pem)
changed: [192.168.11.215] => (item=kubelet-key.pem)
TASK [kube-node : 分发kubeconfig] **************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 准备 cni配置文件] ****************************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-node : 创建kubelet的配置文件] ************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 创建kubelet的systemd unit文件] **************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 开机启用kubelet 服务] ************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 开启kubelet 服务] **************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 分发 kube-proxy.kubeconfig配置文件] **********************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 替换 kube-proxy.kubeconfig 的 apiserver 地址] ***********************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-node : 创建kube-proxy 配置] ***********************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-node : 创建kube-proxy 服务文件] *********************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 开机启用kube-proxy 服务] *********************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-node : 开启kube-proxy 服务] ***********************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 轮询等待kube-proxy启动] **********************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-node : 轮询等待kubelet启动] *************************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
FAILED - RETRYING: 轮询等待node达到Ready状态 (8 retries left).
FAILED - RETRYING: 轮询等待node达到Ready状态 (8 retries left).
TASK [kube-node : 轮询等待node达到Ready状态] *********************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-node : Setting worker role name] **************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
PLAY RECAP ***********************************************************************************************************************************************************************
192.168.11.214 : ok=35 changed=34 unreachable=0 failed=0 skipped=3 rescued=0 ignored=0
192.168.11.215 : ok=35 changed=34 unreachable=0 failed=0 skipped=3 rescued=0 ignored=0
#验证Node节点
[root@K8s-ansible kubeasz]#kubectl get node
NAME STATUS ROLES AGE VERSION
192.168.11.211 Ready,SchedulingDisabled master 27m v1.24.10
192.168.11.212 Ready,SchedulingDisabled master 27m v1.24.10
192.168.11.214 Ready node 2m40s v1.24.10
192.168.11.215 Ready node 2m39s v1.24.10执行kubeasz-setup06-kube-network集群脚本
[root@K8s-ansible kubeasz]#tree roles/calico/
roles/calico/
├── tasks
│ ├── calico-rr.yml
│ └── main.yml
├── templates
│ ├── bgp-default.yaml.j2
│ ├── bgp-rr.yaml.j2
│ ├── calico-csr.json.j2
│ ├── calico-v3.19.yaml.j2
│ ├── calico-v3.23.yaml.j2
│ ├── calico-v3.24.yaml.j2
│ └── calicoctl.cfg.j2
└── vars
└── main.yml
3 directories, 10 files
#执行前更改任务文件的镜像地址
#确认使用的网络插件及其版本
[root@K8s-ansible kubeasz]#cat clusters/k8s-cluster1/config.yml |grep calico
# role:network [flannel,calico,cilium,kube-ovn,kube-router]
# ------------------------------------------- calico
# [calico] IPIP隧道模式可选项有: [Always, CrossSubnet, Never],跨子网可以配置为Always与CrossSubnet(公有云建议使用always比较省事，其他的话需要修改各自公有云的网络配置，具体可以参考各个公有云说明)
# [calico]设置 calico-node使用的host IP，bgp邻居通过该地址建立，可手工指定也可以自动发现
# [calico]设置calico 网络 backend: brid, vxlan, none
# [calico]设置calico 是否使用route reflectors
# [calico]更新支持calico 版本: ["3.19", "3.23"]
calico_ver: "v3.24.5"
#更改任务文件的镜像地址
[root@K8s-ansible kubeasz]#cat roles/calico/templates/calico-v3.24.yaml.j2 |grep image:
image: K8s-harbor01.mooreyxia.com/kubernetes/calico/cni:{{ calico_ver }}
image: K8s-harbor01.mooreyxia.com/kubernetes/calico/node:{{ calico_ver }}
image: K8s-harbor01.mooreyxia.com/kubernetes/calico/node:{{ calico_ver }}
image: K8s-harbor01.mooreyxia.com/kubernetes/calico/kube-controllers:{{ calico_ver }}
[root@K8s-ansible kubeasz]#./ezctl setup k8s-cluster1 06
ansible-playbook -i clusters/k8s-cluster1/hosts -e @clusters/k8s-cluster1/config.yml playbooks/06.network.yml
2023-03-26 14:14:12 INFO cluster:k8s-cluster1 setup step:06 begins in 5s, press any key to abort:
PLAY [kube_master,kube_node] *****************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [192.168.11.212]
ok: [192.168.11.214]
ok: [192.168.11.211]
ok: [192.168.11.215]
TASK [calico : 创建calico 证书请求] ****************************************************************************************************************************************************
changed: [192.168.11.211]
TASK [calico : 创建 calico证书和私钥] ***************************************************************************************************************************************************
changed: [192.168.11.211]
TASK [calico : 删除旧 calico-etcd-secrets] ******************************************************************************************************************************************
changed: [192.168.11.211]
TASK [calico : 创建 calico-etcd-secrets] *******************************************************************************************************************************************
changed: [192.168.11.211]
TASK [calico : 配置 calico DaemonSet yaml文件] ***************************************************************************************************************************************
changed: [192.168.11.211]
TASK [calico : 运行 calico网络] ******************************************************************************************************************************************************
changed: [192.168.11.211]
TASK [calico : 在节点创建相关目录] ********************************************************************************************************************************************************
changed: [192.168.11.215] => (item=/etc/calico/ssl)
changed: [192.168.11.214] => (item=/etc/calico/ssl)
changed: [192.168.11.212] => (item=/etc/calico/ssl)
changed: [192.168.11.211] => (item=/etc/calico/ssl)
TASK [calico : 分发calico证书相关] *****************************************************************************************************************************************************
changed: [192.168.11.212] => (item=ca.pem)
changed: [192.168.11.214] => (item=ca.pem)
changed: [192.168.11.211] => (item=ca.pem)
changed: [192.168.11.215] => (item=ca.pem)
changed: [192.168.11.212] => (item=calico.pem)
changed: [192.168.11.214] => (item=calico.pem)
changed: [192.168.11.211] => (item=calico.pem)
changed: [192.168.11.215] => (item=calico.pem)
changed: [192.168.11.214] => (item=calico-key.pem)
changed: [192.168.11.212] => (item=calico-key.pem)
changed: [192.168.11.211] => (item=calico-key.pem)
changed: [192.168.11.215] => (item=calico-key.pem)
TASK [calico : 删除默认cni配置] ********************************************************************************************************************************************************
changed: [192.168.11.212]
changed: [192.168.11.211]
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [calico : 下载calicoctl 客户端] **************************************************************************************************************************************************
changed: [192.168.11.215] => (item=calicoctl)
changed: [192.168.11.214] => (item=calicoctl)
changed: [192.168.11.211] => (item=calicoctl)
changed: [192.168.11.212] => (item=calicoctl)
TASK [calico : 准备 calicoctl配置文件] *************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
changed: [192.168.11.211]
changed: [192.168.11.212]
FAILED - RETRYING: 轮询等待calico-node 运行 (15 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (15 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (15 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (15 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (14 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (14 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (14 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (14 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (13 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (13 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (13 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (13 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (12 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (12 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (12 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (12 retries left).
TASK [calico : 轮询等待calico-node 运行] ***********************************************************************************************************************************************
changed: [192.168.11.214]
FAILED - RETRYING: 轮询等待calico-node 运行 (11 retries left).
changed: [192.168.11.212]
changed: [192.168.11.211]
changed: [192.168.11.215]
PLAY RECAP ***********************************************************************************************************************************************************************
192.168.11.211 : ok=13 changed=12 unreachable=0 failed=0 skipped=39 rescued=0 ignored=0
192.168.11.212 : ok=7 changed=6 unreachable=0 failed=0 skipped=16 rescued=0 ignored=0
192.168.11.214 : ok=7 changed=6 unreachable=0 failed=0 skipped=16 rescued=0 ignored=0
192.168.11.215 : ok=7 changed=6 unreachable=0 failed=0 skipped=16 rescued=0 ignored=0
#确认网络插件Pod运行中
[root@K8s-ansible kubeasz]#kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-5db878475b-2tqt2 1/1 Running 0 3m25s
kube-system calico-node-bv5qk 1/1 Running 0 3m25s
kube-system calico-node-fb55x 1/1 Running 0 3m25s
kube-system calico-node-j8tgl 1/1 Running 0 3m25s
kube-system calico-node-vxjfl 1/1 Running 0 3m25s测试集群可用性
#创建pod，测试pod网络可用性
[root@K8s-ansible kubeasz]#kubectl create ns myserver
namespace/myserver created
[root@K8s-ansible kubeasz]#kubectl run net-test1 --image=centos:7.9.2009 sleep 100000000 -n myserver
pod/net-test1 created
[root@K8s-ansible kubeasz]#kubectl run net-test2 --image=centos:7.9.2009 sleep 100000000 -n myserver
pod/net-test2 created
[root@K8s-ansible kubeasz]#kubectl run net-test3 --image=centos:7.9.2009 sleep 100000000 -n myserver
pod/net-test3 created
[root@K8s-ansible kubeasz]#kubectl run net-test4 --image=centos:7.9.2009 sleep 100000000 -n myserver
pod/net-test4 created
[root@K8s-ansible kubeasz]#kubectl get pod -n myserver -o wide -w
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
net-test1 1/1 Running 0 50s 10.200.209.2 192.168.11.214
net-test2 1/1 Running 0 45s 10.200.67.2 192.168.11.215
net-test3 1/1 Running 0 40s 10.200.209.3 192.168.11.214
net-test4 1/1 Running 0 35s 10.200.209.4 192.168.11.214
#进去Pod
[root@K8s-ansible ~]#kubectl exec -it net-test1 bash -n myserver
[root@K8s-ansible ~]#kubectl exec -it net-test2 bash -n myserver 
至此一个基本的Kubernetes集群搭建完成
我是moore,大家一起加油！！！
服务器托管，北京服务器托管，服务器租用 http://www.fwqtg.net
机房租用，北京机房租用，IDC机房托管， http://www.e1idc.net
