主机环境预设
#DNS解析
192.168.11.200 K8s-dns.mooreyxia.com DNS01
#镜像仓库
192.168.11.201 K8s-harbor01.mooreyxia.com Harbor01
192.168.11.202 K8s-harbor02.mooreyxia.com Harbor02
#负载均衡器
192.168.11.203 K8s-haproxy01.mooreyxia.com Haproxy01
192.168.11.204 K8s-haproxy02.mooreyxia.com Haproxy02
#Ansible部署节点
192.168.11.205 K8s-ansible.mooreyxia.com Ansible
#Kubernetes控制面板
192.168.11.211 K8s-master01.mooreyxia.com Master01
192.168.11.212 K8s-master02.mooreyxia.com Master02
192.168.11.213 K8s-master03.mooreyxia.com Master03
#Kubernetes工作节点
192.168.11.214 K8s-noded01.mooreyxia.com Node01
192.168.11.215 K8s-noded02.mooreyxia.com Node02
192.168.11.216 K8s-noded03.mooreyxia.com Node03
#Kubernetes分布式存储
192.168.11.217 K8s-etcd01.mooreyxia.com Etcd01
192.168.11.218 K8s-etcd02.mooreyxia.com Etcd02
192.168.11.219 K8s-etcd03.mooreyxia.com Etcd03
#VIP
192.168.11.241~245
#KeepAlived01~02心跳线
192.168.11.246~247部署DNS服务
安装DNS服务并设置开机启动
[root@k8s-dns ~]#vim install_dns.sh
[root@k8s-dns ~]#cat install_dns.sh
#!/bin/bash
DOMAIN=mooreyxia.com
HOST=K8s-dns
HOST_IP=192.168.11.200
LOCALHOST=`hostname -I | awk '{print $1}'`
. /etc/os-release
color () {
RES_COL=60
MOVE_TO_COL="echo -en \033[${RES_COL}G"
SETCOLOR_SUCCESS="echo -en \033[1;32m"
SETCOLOR_FAILURE="echo -en \033[1;31m"
SETCOLOR_WARNING="echo -en \033[1;33m"
SETCOLOR_NORMAL="echo -en E[0m"
echo -n "$1" && $MOVE_TO_COL
echo -n "["
if [ $2 = "success" -o $2 = "0" ] ;then
${SETCOLOR_SUCCESS}
echo -n $" OK "
elif [ $2 = "failure" -o $2 = "1" ] ;then
${SETCOLOR_FAILURE}
echo -n $"FAILED"
else
${SETCOLOR_WARNING}
echo -n $"WARNING"
fi
${SETCOLOR_NORMAL}
echo -n "]"
echo
}
install_dns () {
if [ $ID = 'centos' -o $ID = 'rocky' ];then
yum install -y bind bind-utils
elif [ $ID = 'ubuntu' ];then
apt update
apt install -y bind9 bind9-utils bind9-host bind9-dnsutils
else
color "不支持此操作系统，退出!" 1
exit
fi
}
config_dns () {
if [ $ID = 'centos' -o $ID = 'rocky' ];then
sed -i -e '/listen-on/s/127.0.0.1/localhost/' -e '/allow-query/s/localhost/any/' -e 's/dnssec-enable yes/dnssec-enable no/' -e 's/dnssec-validation yes/dnssec-validation no/' /etc/named.conf
cat >> /etc/named.rfc1912.zones /var/named/$DOMAIN.zone > /etc/bind/named.conf.default-zones /etc/bind/$DOMAIN.zone 添加预设DNS解析
#确认域名解析
[root@k8s-dns ~]#cat /etc/bind/named.conf.default-zones |grep mooreyxia.com
zone "mooreyxia.com" IN {
file "/etc/bind/mooreyxia.com.zone";
#添加子域
[root@k8s-dns ~]#vim /etc/bind/mooreyxia.com.zone
[root@k8s-dns ~]#cat /etc/bind/mooreyxia.com.zone
$TTL 1D
@ IN SOA master admin (
1 ; serial
1D ; refresh
1H ; retry
1W ; expire
3H ) ; minimum
NS master
master A 192.168.11.200
k8s-dns A 192.168.11.200
K8s-harbor01 A 192.168.11.201
K8s-harbor02 A 192.168.11.202
K8s-haproxy01 A 192.168.11.203
K8s-haproxy02 A 192.168.11.204
K8s-master01 A 192.168.11.211
K8s-master02 A 192.168.11.212
K8s-master03 A 192.168.11.213
K8s-noded01 A 192.168.11.214
K8s-noded02 A 192.168.11.215
K8s-noded03 A 192.168.11.216
K8s-etcd01 A 192.168.11.217
K8s-etcd02 A 192.168.11.218
K8s-etcd03 A 192.168.11.219
[root@k8s-dns ~]#systemctl restart named
#将DNS解析地址指向自己
[root@k8s-dns ~]#vim /etc/netplan/01-netcfg.yaml
[root@k8s-dns ~]#cat /etc/netplan/01-netcfg.yaml
# This file describes the network interfaces available on your system
# For more information, see netplan(5).
network:
version: 2
renderer: networkd
ethernets:
eth0:
addresses:
- 192.168.11.200/24
gateway4: 192.168.11.1
nameservers:
search: [mooreyxia.org, mooreyxia.com]
addresses: [127.0.0.1]
#重新加载网卡配置
[root@k8s-dns ~]#netplan apply
** (generate:1668): WARNING **: 11:51:57.037: `gateway4` has been deprecated, use default routes instead.
See the 'Default routes' section of the documentation for more details.
** (process:1666): WARNING **: 11:51:57.585: `gateway4` has been deprecated, use default routes instead.
See the 'Default routes' section of the documentation for more details.
#测试
[root@k8s-dns ~]#dig k8s-dns.mooreyxia.com
; > DiG 9.18.1-1ubuntu1.3-Ubuntu > k8s-dns.mooreyxia.com
;; global options: +cmd
;; Got answer:
;; ->>HEADER> DiG 9.18.1-1ubuntu1.3-Ubuntu > K8s-harbor01.mooreyxia.com
;; global options: +cmd
;; Got answer:
;; ->>HEADER部署Harbor并实现https(SAN签发证书)
配置DNS解析
#将主机DNS解释地址指向DNS服务器，后续服务器都同样操作，不再赘述
[root@K8s-harbor01 ~]#hostname
K8s-harbor01.mooreyxia.com
[root@K8s-harbor01 ~]#vim /etc/netplan/01-netcfg.yaml
[root@K8s-harbor01 ~]#cat /etc/netplan/01-netcfg.yaml
# This file describes the network interfaces available on your system
# For more information, see netplan(5).
network:
version: 2
renderer: networkd
ethernets:
eth0:
addresses:
- 192.168.11.201/24
gateway4: 192.168.11.1
nameservers:
search: [mooreyxia.org, mooreyxia.com]
addresses: [192.168.11.200]
[root@K8s-harbor01 ~]#netplan apply
** (generate:907): WARNING **: 11:56:40.788: `gateway4` has been deprecated, use default routes instead.
See the 'Default routes' section of the documentation for more details.
** (process:905): WARNING **: 11:56:41.357: `gateway4` has been deprecated, use default routes instead.
See the 'Default routes' section of the documentation for more details.安装 Docker
[root@K8s-harbor01 ~]#cat install_docker.sh
#!/bin/bash
DOCKER_VERSION="23.0.1"
#DOCKER_VERSION="20.10.10"
DOCKER_URL="http://mirrors.ustc.edu.cn"
#DOCKER_URL="https://mirrors.aliyun.com"
#DOCKER_URL="https://mirrors.tuna.tsinghua.edu.cn"
COLOR_SUCCESS="echo -e \033[1;32m"
COLOR_FAILURE="echo -e \033[1;31m"
END=" 33[m"
. /etc/os-release
UBUNTU_DOCKER_VERSION="5:${DOCKER_VERSION}-1~ubuntu.22.04~${UBUNTU_CODENAME}"
#UBUNTU_DOCKER_VERSION="5:${DOCKER_VERSION}~3-0~${ID}-${UBUNTU_CODENAME}"
#UBUNTU_DOCKER_VERSION="5:20.10.9~3-0~`lsb_release -si`-`lsb_release -cs`"
#UBUNTU_DOCKER_VERSION="5:19.03.14~3-0~lsb_release -si-`lsb_release -cs`"
color () {
RES_COL=60
MOVE_TO_COL="echo -en \033[${RES_COL}G"
SETCOLOR_SUCCESS="echo -en \033[1;32m"
SETCOLOR_FAILURE="echo -en \033[1;31m"
SETCOLOR_WARNING="echo -en \033[1;33m"
SETCOLOR_NORMAL="echo -en E[0m"
echo -n "$1" && $MOVE_TO_COL
echo -n "["
if [ $2 = "success" -o $2 = "0" ] ;then
${SETCOLOR_SUCCESS}
echo -n $" OK "
elif [ $2 = "failure" -o $2 = "1" ] ;then
${SETCOLOR_FAILURE}
echo -n $"FAILED"
else
${SETCOLOR_WARNING}
echo -n $"WARNING"
fi
${SETCOLOR_NORMAL}
echo -n "]"
echo
}
install_docker(){
if [ $ID = "centos" -o $ID = "rocky" ];then
if [ $VERSION_ID = "7" ];then
cat > /etc/yum.repos.d/docker.repo /etc/yum.repos.d/docker.repo /dev/null && $COLOR"Docker已安装，退出" 1 && exit
apt update || { color "更新包索引失败" 1 ; exit 1; }
apt -y install apt-transport-https ca-certificates curl software-properties-common ||
{ color "安装相关包失败" 1 ; exit 2; }
curl -fsSL ${DOCKER_URL}/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
add-apt-repository "deb [arch=amd64] ${DOCKER_URL}/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
apt update
${COLOR_FAILURE} "Docker有以下版本"${END}
apt-cache madison docker-ce
${COLOR_FAILURE}"5秒后即将安装: docker-"${UBUNTU_DOCKER_VERSION}" 版本....."${END}
${COLOR_FAILURE}"如果想安装其它Docker版本，请按ctrl+c键退出，修改版本再执行"${END}
sleep 5
apt -y install docker-ce=${UBUNTU_DOCKER_VERSION} docker-ce-cli=${UBUNTU_DOCKER_VERSION}
fi
if [ $? -eq 0 ];then
color "安装软件包成功" 0
else
color "安装软件包失败，请检查网络配置" 1
exit
fi
}
config_docker (){
mkdir -p /etc/docker
tee /etc/docker/daemon.json 安装 Docker compose
[root@K8s-harbor01 ~]#curl -L https://get.daocloud.io/docker/compose/releases/download/v2.16.0/docker-compose-$(uname -s)-$(uname -m) -o /usr/bin/docker-compose
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 145 100 145 0 0 572 0 --:--:-- --:--:-- --:--:-- 573
100 18906 100 18906 0 0 4993 0 0:00:03 0:00:03 --:--:-- 6490
[root@K8s-harbor01 ~]#chmod +x /usr/bin/docker-compose
[root@K8s-harbor01 ~]#docker-compose --version
Illegal option --version
# Executing docker install script, commit: 66474034547a96caa0a25be56051ff8b726a1b28
Warning: the "docker" command appears to already exist on this system.
If you already have Docker installed, this script can cause trouble, which is
why we're displaying this warning and provide the opportunity to cancel the
installation.
If you installed the current Docker package using this script and are using it
again to update Docker, you can safely ignore this message.
You may press Ctrl+C now to abort this script.
+ sleep 20
+ sh -c apt-get update -qq >/dev/null
W: http://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu/dists/jammy/InRelease: Key is stored in legacy trusted.gpg keyring (/etc/apt/trusted.gpg), see the DEPRECATION section in apt-key(8) for details.
+ sh -c DEBIAN_FRONTEND=noninteractive apt-get install -y -qq apt-transport-https ca-certificates curl >/dev/null
+ sh -c mkdir -p /etc/apt/keyrings && chmod -R 0755 /etc/apt/keyrings
+ sh -c curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
+ sh -c chmod a+r /etc/apt/keyrings/docker.gpg
+ sh -c echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu jammy stable" > /etc/apt/sources.list.d/docker.list
+ sh -c apt-get update -qq >/dev/null
W: http://mirrors.ustc.edu.cn/docker-ce/linux/ubuntu/dists/jammy/InRelease: Key is stored in legacy trusted.gpg keyring (/etc/apt/trusted.gpg), see the DEPRECATION section in apt-key(8) for details.
+ sh -c DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-scan-plugin docker-compose-plugin docker-ce-rootless-extras docker-buildx-plugin >/dev/null
+ sh -c docker version
Client: Docker Engine - Community
Version: 23.0.1
API version: 1.42
Go version: go1.19.5
Git commit: a5ee5b1
Built: Thu Feb 9 19:47:01 2023
OS/Arch: linux/amd64
Context: default
Server: Docker Engine - Community
Engine:
Version: 23.0.1
API version: 1.42 (minimum version 1.12)
Go version: go1.19.5
Git commit: bc3805a
Built: Thu Feb 9 19:47:01 2023
OS/Arch: linux/amd64
Experimental: false
containerd:
Version: 1.6.18
GitCommit: 2456e983eb9e37e47538f59ea18f2043c9a73640
runc:
Version: 1.1.4
GitCommit: v1.1.4-0-g5fd4c4d
docker-init:
Version: 0.19.0
GitCommit: de40ad0安装 Harbor
[root@K8s-harbor01 ~]#mkdir /apps
[root@K8s-harbor01 ~]#tar xvf harbor-offline-installer-v1.7.6.tgz -C /apps/
[root@K8s-harbor01 ~]#cd /apps/harbor
[root@K8s-harbor01 ~]#cp harbor.yml.tmpl harbor.yml
[root@K8s-harbor01 ~]#sed -ri "/^hostname/s/reg.mydomain.com/${HARBOR_NAME}/" harbor.yml
[root@K8s-harbor01 ~]#sed -ri "/^https/s/(https:)/#1/" harbor.yml
[root@K8s-harbor01 ~]#sed -ri "s/(port: 443)/#1/" harbor.yml
[root@K8s-harbor01 ~]#sed -ri "/certificate:/s/(.*)/#1/" harbor.yml
[root@K8s-harbor01 ~]#sed -ri "/private_key:/s/(.*)/#1/" harbor.yml
[root@K8s-harbor01 ~]#sed -ri "s/Harbor12345/${HARBOR_ADMIN_PASSWORD}/" harbor.yml
[root@K8s-harbor01 ~]#sed -i 's#^data_volume: /data#data_volume: /data/harbor#' harbor.yml
[root@K8s-harbor01 ~]#. /apps//harbor/install.sh
harbor/harbor.v2.7.1.tar.gz
harbor/prepare
harbor/LICENSE
harbor/install.sh
harbor/common.sh
harbor/harbor.yml.tmpl
[Step 0]: checking if docker is installed ...
Note: docker version: 23.0.1
[Step 1]: checking docker-compose is installed ...
Note: Docker Compose version v2.16.0
[Step 2]: loading Harbor images ...
d635f8a69c64: Loading layer [==================================================>] 37.73MB/37.73MB
9b681d7437ee: Loading layer [==================================================>] 5.758MB/5.758MB
a83e69e7a5ba: Loading layer [==================================================>] 8.987MB/8.987MB
4976a6e767f1: Loading layer [==================================================>] 14.47MB/14.47MB
480d367aea76: Loading layer [==================================================>] 29.29MB/29.29MB
dee8463c1f24: Loading layer [==================================================>] 22.02kB/22.02kB
54570f91d11d: Loading layer [==================================================>] 14.47MB/14.47MB
Loaded image: goharbor/notary-signer-photon:v2.7.1
1f4c775bffe8: Loading layer [==================================================>] 5.758MB/5.758MB
3284ab8d01eb: Loading layer [==================================================>] 8.987MB/8.987MB
dd8fa1a2b2b0: Loading layer [==================================================>] 15.88MB/15.88MB
8106ce046abf: Loading layer [==================================================>] 29.29MB/29.29MB
1053b310e287: Loading layer [==================================================>] 22.02kB/22.02kB
0aa0bd6cf9e6: Loading layer [==================================================>] 15.88MB/15.88MB
Loaded image: goharbor/notary-server-photon:v2.7.1
29f59c2a2eb6: Loading layer [==================================================>] 43.64MB/43.64MB
b27d3e5a8df6: Loading layer [==================================================>] 65.83MB/65.83MB
30a3023736ef: Loading layer [==================================================>] 19.19MB/19.19MB
9b3adbbdba09: Loading layer [==================================================>] 65.54kB/65.54kB
31692c2b302f: Loading layer [==================================================>] 2.56kB/2.56kB
c97591717d77: Loading layer [==================================================>] 1.536kB/1.536kB
447e7da95733: Loading layer [==================================================>] 12.29kB/12.29kB
9215e490929e: Loading layer [==================================================>] 2.614MB/2.614MB
f02c03ccd4ea: Loading layer [==================================================>] 407kB/407kB
Loaded image: goharbor/prepare:v2.7.1
7a5dc411df76: Loading layer [==================================================>] 91.1MB/91.1MB
ceff2ad73365: Loading layer [==================================================>] 6.145MB/6.145MB
2abd5823ecd7: Loading layer [==================================================>] 1.249MB/1.249MB
e30926f66daa: Loading layer [==================================================>] 1.194MB/1.194MB
Loaded image: goharbor/harbor-portal:v2.7.1
3a104ad3dfa8: Loading layer [==================================================>] 123.4MB/123.4MB
a582b782d956: Loading layer [==================================================>] 17.67MB/17.67MB
747ad719967e: Loading layer [==================================================>] 5.12kB/5.12kB
347857745db7: Loading layer [==================================================>] 6.144kB/6.144kB
7ffc6a14a878: Loading layer [==================================================>] 3.072kB/3.072kB
40e4f5237627: Loading layer [==================================================>] 2.048kB/2.048kB
115d9d0765bb: Loading layer [==================================================>] 2.56kB/2.56kB
b707c713f224: Loading layer [==================================================>] 2.56kB/2.56kB
60dba025610e: Loading layer [==================================================>] 2.56kB/2.56kB
4ccd0988c811: Loading layer [==================================================>] 9.728kB/9.728kB
Loaded image: goharbor/harbor-db:v2.7.1
cb897960d30f: Loading layer [==================================================>] 8.906MB/8.906MB
fb6f4fd16c90: Loading layer [==================================================>] 3.584kB/3.584kB
173d6ab78d38: Loading layer [==================================================>] 2.56kB/2.56kB
c1fc6a63abb3: Loading layer [==================================================>] 103.2MB/103.2MB
5f7e1bd6f6a5: Loading layer [==================================================>] 104MB/104MB
Loaded image: goharbor/harbor-jobservice:v2.7.1
856cff93e4d6: Loading layer [==================================================>] 8.906MB/8.906MB
cf9e05a3f1f1: Loading layer [==================================================>] 25.65MB/25.65MB
7bccc4373df9: Loading layer [==================================================>] 4.608kB/4.608kB
b3ca201ee095: Loading layer [==================================================>] 26.44MB/26.44MB
Loaded image: goharbor/harbor-exporter:v2.7.1
9023a432c05c: Loading layer [==================================================>] 91.1MB/91.1MB
Loaded image: goharbor/nginx-photon:v2.7.1
4f730363c02d: Loading layer [==================================================>] 5.764MB/5.764MB
23e70dd83173: Loading layer [==================================================>] 4.096kB/4.096kB
a42372d3c73d: Loading layer [==================================================>] 3.072kB/3.072kB
a76104dd55c5: Loading layer [==================================================>] 17.41MB/17.41MB
77cd695198ba: Loading layer [==================================================>] 18.2MB/18.2MB
Loaded image: goharbor/registry-photon:v2.7.1
ad617d2bc63f: Loading layer [==================================================>] 6.291MB/6.291MB
77e676fefcfc: Loading layer [==================================================>] 4.096kB/4.096kB
2748a8bff421: Loading layer [==================================================>] 3.072kB/3.072kB
e44fb65f4eba: Loading layer [==================================================>] 185.6MB/185.6MB
97cc74bfda1f: Loading layer [==================================================>] 13.73MB/13.73MB
928b80c07fce: Loading layer [==================================================>] 200.1MB/200.1MB
Loaded image: goharbor/trivy-adapter-photon:v2.7.1
09859c8fc317: Loading layer [==================================================>] 5.763MB/5.763MB
a2262dec17c9: Loading layer [==================================================>] 91.75MB/91.75MB
1cda8d191cea: Loading layer [==================================================>] 3.072kB/3.072kB
6015205cdf66: Loading layer [==================================================>] 4.096kB/4.096kB
7197ebd96a1d: Loading layer [==================================================>] 92.54MB/92.54MB
Loaded image: goharbor/chartmuseum-photon:v2.7.1
1f5497d46044: Loading layer [==================================================>] 8.906MB/8.906MB
af93c68750db: Loading layer [==================================================>] 3.584kB/3.584kB
c5cf9feadf24: Loading layer [==================================================>] 2.56kB/2.56kB
26a09c87d905: Loading layer [==================================================>] 84.83MB/84.83MB
1347fb69ddac: Loading layer [==================================================>] 5.632kB/5.632kB
4b368dfd8acc: Loading layer [==================================================>] 108kB/108kB
6a175467b77e: Loading layer [==================================================>] 44.03kB/44.03kB
62281197b07e: Loading layer [==================================================>] 85.78MB/85.78MB
cdcda83640cc: Loading layer [==================================================>] 2.56kB/2.56kB
Loaded image: goharbor/harbor-core:v2.7.1
0d6e31ae4572: Loading layer [==================================================>] 99MB/99MB
64ae4b9c8bce: Loading layer [==================================================>] 3.584kB/3.584kB
c6a1a03ae721: Loading layer [==================================================>] 3.072kB/3.072kB
4b8ff43d0c5f: Loading layer [==================================================>] 2.56kB/2.56kB
a4ce38a3c8a8: Loading layer [==================================================>] 3.072kB/3.072kB
759b81f2ba78: Loading layer [==================================================>] 3.584kB/3.584kB
fecfd1ab18de: Loading layer [==================================================>] 20.99kB/20.99kB
Loaded image: goharbor/harbor-log:v2.7.1
e04fe2da5f19: Loading layer [==================================================>] 5.764MB/5.764MB
fc3ccce61cbe: Loading layer [==================================================>] 4.096kB/4.096kB
e1befb3b20d0: Loading layer [==================================================>] 17.41MB/17.41MB
381df6c03446: Loading layer [==================================================>] 3.072kB/3.072kB
b819771c362f: Loading layer [==================================================>] 30.69MB/30.69MB
c508d3c8aeac: Loading layer [==================================================>] 48.89MB/48.89MB
Loaded image: goharbor/harbor-registryctl:v2.7.1
c36fbbc4c25b: Loading layer [==================================================>] 91.93MB/91.93MB
c77d114477fc: Loading layer [==================================================>] 3.072kB/3.072kB
b5adfac82a36: Loading layer [==================================================>] 59.9kB/59.9kB
1161cc44e0d4: Loading layer [==================================================>] 61.95kB/61.95kB
Loaded image: goharbor/redis-photon:v2.7.1
[Step 3]: preparing environment ...
[Step 4]: preparing harbor configs ...
prepare base dir is set to /apps/harbor
WARNING:root:WARNING: HTTP protocol is insecure. Harbor will deprecate http protocol in the future. Please make sure to upgrade to https
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
Generated and saved secret to file: /data/secret/keys/secretkey
Successfully called func: create_root_cert
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
Note: stopping existing Harbor instance ...
[Step 5]: starting Harbor ...
[+] Running 10/10
⠿ Network harbor_harbor Created 0.1s
⠿ Container harbor-log Started 2.8s
⠿ Container redis Started 4.3s
⠿ Container harbor-db Started 4.2s
⠿ Container harbor-portal Started 4.7s
⠿ Container registryctl Started 4.6s
⠿ Container registry Started 4.1s
⠿ Container harbor-core Started 5.0s
⠿ Container nginx Started 6.2s
⠿ Container harbor-jobservice Started 6.3s
#设置service，并开机启动
cat > /lib/systemd/system/harbor.service 
Harbor 安全 Https 配置
- 生成 Harbor 服务器证书
#建立一个存放证书的目录
[root@K8s-harbor02 ~]#mkdir -p /data/harbor/certs
[root@K8s-harbor02 ~]#cd /data/harbor/certs
[root@K8s-harbor02 certs]#ls
#生成ca私钥
[root@K8s-harbor02 certs]#openssl genrsa -out ca.key 4096
#生成ca的自签名证书【域名自定义】
[root@K8s-harbor02 certs]#openssl req -x509 -new -nodes -sha512 -days 3650
-subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=mooreyxia.com"
-key ca.key
-out ca.crt
#生成harbor主机的私钥【主机域名要确保正确】
[root@K8s-harbor02 certs]#openssl genrsa -out K8s-harbor02.mooreyxia.com.key 4096
#生成harbor主机的证书申请【主机域名要确保正确】
[root@K8s-harbor02 certs]#openssl req -sha512 -new
-subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=K8s-harbor02.mooreyxia.com"
-key K8s-harbor02.mooreyxia.com.key
-out K8s-harbor02.mooreyxia.com.csr
[root@K8s-harbor02 certs]#ls
K8s-harbor02.mooreyxia.com.csr K8s-harbor02.mooreyxia.com.key ca.crt ca.key privkey.pem
#创建x509 v3 扩展文件(新版新增加的要求)
[root@K8s-harbor02 certs]#cat > v3.ext - 配置 Harbor 服务器使用证书
[root@K8s-harbor02 certs]#vim /apps/harbor/harbor.yml
[root@K8s-harbor02 certs]#cat /apps/harbor/harbor.yml
# Configuration file of Harbor
# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: K8s-harbor02.mooreyxia.com
# http related config
http:
# port for http, default is 80. If https enabled, this port will redirect to https port
port: 80
# https related config
https:
# https port for harbor, default is 443
port: 443
# The path of cert and key files for nginx
certificate: /data/harbor/certs/K8s-harbor02.mooreyxia.com.crt
private_key: /data/harbor/certs/K8s-harbor02.mooreyxia.com.key
....
#保存退出配置文件后使配置生效
[root@K8s-harbor02 certs]#cd /apps/harbor/
[root@K8s-harbor02 harbor]#./prepare
prepare base dir is set to /apps/harbor
Clearing the configuration file: /config/core/env
Clearing the configuration file: /config/core/app.conf
Clearing the configuration file: /config/nginx/nginx.conf
Clearing the configuration file: /config/jobservice/env
Clearing the configuration file: /config/jobservice/config.yml
Clearing the configuration file: /config/log/logrotate.conf
Clearing the configuration file: /config/log/rsyslog_docker.conf
Clearing the configuration file: /config/registryctl/env
Clearing the configuration file: /config/registryctl/config.yml
Clearing the configuration file: /config/portal/nginx.conf
Clearing the configuration file: /config/db/env
Clearing the configuration file: /config/registry/root.crt
Clearing the configuration file: /config/registry/passwd
Clearing the configuration file: /config/registry/config.yml
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
loaded secret from file: /data/secret/keys/secretkey
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
#重新启动docker-compose
[root@K8s-harbor02 harbor]#docker-compose down -v
Stopping harbor-jobservice ... done
Stopping nginx ... done
Stopping harbor-core ... done
Stopping harbor-db ... done
Stopping registryctl ... done
Stopping redis ... done
Stopping registry ... done
Stopping harbor-portal ... done
Stopping harbor-log ... done
Removing harbor-jobservice ... done
Removing nginx ... done
Removing harbor-core ... done
Removing harbor-db ... done
Removing registryctl ... done
Removing redis ... done
Removing registry ... done
Removing harbor-portal ... done
Removing harbor-log ... done
Removing network harbor_harbor
[root@K8s-harbor02 harbor]#docker-compose up -d
Creating network "harbor_harbor" with the default driver
Creating harbor-log ... done
Creating registry ... done
Creating harbor-portal ... done
Creating harbor-db ... done
Creating redis ... done
Creating registryctl ... done
Creating harbor-core ... done
Creating nginx ... done
Creating harbor-jobservice ... done
#输入下面 http 链接自动跳转到 https
https://K8s-harbor02.mooreyxia.com
实现 Harbor 高可用（双向复制）
- harbor01主机和harbor02主机建立同样名称的项目文件
- harbor01主机和harbor02主机以对方为目标互相建立复制目标
- harbor01主机和harbor02主机以对方为目标互相建立复制策略
复制证书给其他主机并上传镜像
- 生成授权证书文件
[root@K8s-harbor01 ~]#cd /data/harbor/certs/
[root@K8s-harbor01 certs]#ls
K8s-harbor01.mooreyxia.com.crt K8s-harbor01.mooreyxia.com.csr K8s-harbor01.mooreyxia.com.key ca.crt ca.key v3.ext
#转换harbor的crt证书文件为cert后缀,docker识别crt文件为CA证书,cert为客户端证书服务器
[root@K8s-harbor01 certs]#openssl x509 -inform PEM -in K8s-harbor01.mooreyxia.com.crt -out K8s-harbor01.mooreyxia.com.cert
[root@K8s-harbor01 certs]#ls
K8s-harbor01.mooreyxia.com.cert K8s-harbor01.mooreyxia.com.crt K8s-harbor01.mooreyxia.com.csr K8s-harbor01.mooreyxia.com.key ca.crt ca.key v3.ext
#比较两个文件的不同
[root@K8s-harbor01 certs]#cat K8s-harbor01.mooreyxia.com.crt
-----BEGIN CERTIFICATE-----
MIIGKTCCBBGgAwIBAgIUUAs+VRTNIuKNxMZOUTaKxKwF7towDQYJKoZIhvcNAQEN
BQAwbjELMAkGA1UEBhMCQ04xEDAOBgNVBAgMB0JlaWppbmcxEDAOBgNVBAcMB0Jl
aWppbmcxEDAOBgNVBAoMB2V4YW1wbGUxETAPBgNVBAsMCFBlcnNvbmFsMRYwFAYD
VQQDDA1tb29yZXl4aWEuY29tMB4XDTIzMDMxODA0NTkwM1oXDTMzMDMxNTA0NTkw
M1owezELMAkGA1UEBhMCQ04xEDAOBgNVBAgMB0JlaWppbmcxEDAOBgNVBAcMB0Jl
aWppbmcxEDAOBgNVBAoMB2V4YW1wbGUxETAPBgNVBAsMCFBlcnNvbmFsMSMwIQYD
VQQDDBpLOHMtaGFyYm9yMDEubW9vcmV5eGlhLmNvbTCCAiIwDQYJKoZIhvcNAQEB
BQADggIPADCCAgoCggIBALgjKIrgbN1Idk8jbZNDqt/TXpA7kxFbnr0vbxX1cjRa
nSm5Juahf6adQ++Zn4NTEG4ZHN4lb6PKiSn7cBNOsQn8NuM3qy42PJcZQZC6JPiz
ag2K+8Ec5DDCWpni+m9IQVrH8vPFEzhzE7gLu2KmdCUZnKRTNthlUtI9wIjonPv9
FhMrBw6BKMO3BqTZvM82kXf/eAooxTNof9/BgLBYgOl5hroDepk8+ZPf/okc2qs+
DiAbOXyz3pU7VVmci4FlFSMbG1kI6RCRLQ8as8rYRd3TIAwoQDovXpopR6Eotr+9
Yrra9r9+z0eb2wJ4eaoYq9fXityAngVGMAJX0BZWRpS6l2j1a2ihN60qWBT3ZIvu
QWlv8ElMYJ/EruHTwF3FTRAwB1/byjD51oNRHe9QbEolTiNpwojI8ZTv5k8g9Irf
T+pbdO9KJu5RxTH1M1rQ5nYRT81uIKafZ7IhHGUnxqTSZRie7yKvPwbUZgr++VRp
S+5Nnm9uy/8uZHcGDQhzMD6YF3WC/IiJYDvudlzyAqr173oKvVD7uQG+28F0fWpn
oAqMexz+Yqs1ozX+HU/XO5yaUl4AtQFpSOaotl3PhaoE+qU+4Rq03a0lyq001PVi
T8VkvLNbOmOkZ8c4SStE1U7M6o1+EqgvdHxdQx3KJqFp+amYk2f6iGwCUrVuX1AJ
AgMBAAGjgbEwga4wHwYDVR0jBBgwFoAUEVa5KVB9bYko5KWBUEBDbaTm7jwwCQYD
VR0TBAIwADALBgNVHQ8EBAMCBPAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwPwYDVR0R
BDgwNoINbW9vcmV5eGlhLmNvbYIJbW9vcmV5eGlhghpLOHMtaGFyYm9yMDEubW9v
cmV5eGlhLmNvbTAdBgNVHQ4EFgQUaFK6Wa6blRt0vo2DoLjRLfP9k5owDQYJKoZI
hvcNAQENBQADggIBAIkESdVU8V6U0Qg/aQgAqacfeSBHWLNUlHfZ5QAf2p4j4KWN
Lqy8DEDWYyheuAp50bxPWiqpHDYzvMucJw8B8h1SmE82xGdr0j6d3Loj9M1O5bqY
SyoK5yzN62Xn/dCQjuLI9Ib42VG7SBQ5E2Xyf8u3gnoGyF9AaK2BUa1KrvBus9Yb
yEUO1Y97dQM78a54BX0f7+MaRFAFAUdRBrO/vblcECCatCD8PQIIk+8zpOG/M26T
ijqHVsN0e5/o8m+1gFxqh+LuR9W2NMGKy+WX3c9HYwprnUwyX+CPHhknHEoPha0u
CkfLEfED9dQt9zTb54o5ykTEG9vJmV4oex+7pwZVmsuJAjgMVxw5XfBTwAhJVzf4
s3mu+BDaKqiGmzeIQZaxjdCooRw33cnXmRnPZQ0O2kR9WKzpSFsuXXKTmkglPsV0
8VEtbiBeKhb3+OfWPNndQgWSm3g6dQ1X2cUhbsyT8S7orQEwyi2ZX1eM8qBmF+0P
0KuhzwHISbD9TmyfzpIcZ6yWW6VU5eOrz+lAGjH96gaoV+Cu/9o9Z9nzKl2SCNUt
3mni1/yJIOoq0N3Yvu8Kq9qHHXWGhaaVEBuI802B+SGizGretMR7AMgVMGYNHp3d
APS2/baaP2P29w7tMwVE1IwnMBVVnfFAV62U+dbhxuqzFJP1bgpngUYVg1Vv
-----END CERTIFICATE-----
[root@K8s-harbor01 certs]#cat K8s-harbor01.mooreyxia.com.cert
-----BEGIN CERTIFICATE-----
MIIGKTCCBBGgAwIBAgIUUAs+VRTNIuKNxMZOUTaKxKwF7towDQYJKoZIhvcNAQEN
BQAwbjELMAkGA1UEBhMCQ04xEDAOBgNVBAgMB0JlaWppbmcxEDAOBgNVBAcMB0Jl
aWppbmcxEDAOBgNVBAoMB2V4YW1wbGUxETAPBgNVBAsMCFBlcnNvbmFsMRYwFAYD
VQQDDA1tb29yZXl4aWEuY29tMB4XDTIzMDMxODA0NTkwM1oXDTMzMDMxNTA0NTkw
M1owezELMAkGA1UEBhMCQ04xEDAOBgNVBAgMB0JlaWppbmcxEDAOBgNVBAcMB0Jl
aWppbmcxEDAOBgNVBAoMB2V4YW1wbGUxETAPBgNVBAsMCFBlcnNvbmFsMSMwIQYD
VQQDDBpLOHMtaGFyYm9yMDEubW9vcmV5eGlhLmNvbTCCAiIwDQYJKoZIhvcNAQEB
BQADggIPADCCAgoCggIBALgjKIrgbN1Idk8jbZNDqt/TXpA7kxFbnr0vbxX1cjRa
nSm5Juahf6adQ++Zn4NTEG4ZHN4lb6PKiSn7cBNOsQn8NuM3qy42PJcZQZC6JPiz
ag2K+8Ec5DDCWpni+m9IQVrH8vPFEzhzE7gLu2KmdCUZnKRTNthlUtI9wIjonPv9
FhMrBw6BKMO3BqTZvM82kXf/eAooxTNof9/BgLBYgOl5hroDepk8+ZPf/okc2qs+
DiAbOXyz3pU7VVmci4FlFSMbG1kI6RCRLQ8as8rYRd3TIAwoQDovXpopR6Eotr+9
Yrra9r9+z0eb2wJ4eaoYq9fXityAngVGMAJX0BZWRpS6l2j1a2ihN60qWBT3ZIvu
QWlv8ElMYJ/EruHTwF3FTRAwB1/byjD51oNRHe9QbEolTiNpwojI8ZTv5k8g9Irf
T+pbdO9KJu5RxTH1M1rQ5nYRT81uIKafZ7IhHGUnxqTSZRie7yKvPwbUZgr++VRp
S+5Nnm9uy/8uZHcGDQhzMD6YF3WC/IiJYDvudlzyAqr173oKvVD7uQG+28F0fWpn
oAqMexz+Yqs1ozX+HU/XO5yaUl4AtQFpSOaotl3PhaoE+qU+4Rq03a0lyq001PVi
T8VkvLNbOmOkZ8c4SStE1U7M6o1+EqgvdHxdQx3KJqFp+amYk2f6iGwCUrVuX1AJ
AgMBAAGjgbEwga4wHwYDVR0jBBgwFoAUEVa5KVB9bYko5KWBUEBDbaTm7jwwCQYD
VR0TBAIwADALBgNVHQ8EBAMCBPAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwPwYDVR0R
BDgwNoINbW9vcmV5eGlhLmNvbYIJbW9vcmV5eGlhghpLOHMtaGFyYm9yMDEubW9v
cmV5eGlhLmNvbTAdBgNVHQ4EFgQUaFK6Wa6blRt0vo2DoLjRLfP9k5owDQYJKoZI
hvcNAQENBQADggIBAIkESdVU8V6U0Qg/aQgAqacfeSBHWLNUlHfZ5QAf2p4j4KWN
Lqy8DEDWYyheuAp50bxPWiqpHDYzvMucJw8B8h1SmE82xGdr0j6d3Loj9M1O5bqY
SyoK5yzN62Xn/dCQjuLI9Ib42VG7SBQ5E2Xyf8u3gnoGyF9AaK2BUa1KrvBus9Yb
yEUO1Y97dQM78a54BX0f7+MaRFAFAUdRBrO/vblcECCatCD8PQIIk+8zpOG/M26T
ijqHVsN0e5/o8m+1gFxqh+LuR9W2NMGKy+WX3c9HYwprnUwyX+CPHhknHEoPha0u
CkfLEfED9dQt9zTb54o5ykTEG9vJmV4oex+7pwZVmsuJAjgMVxw5XfBTwAhJVzf4
s3mu+BDaKqiGmzeIQZaxjdCooRw33cnXmRnPZQ0O2kR9WKzpSFsuXXKTmkglPsV0
8VEtbiBeKhb3+OfWPNndQgWSm3g6dQ1X2cUhbsyT8S7orQEwyi2ZX1eM8qBmF+0P
0KuhzwHISbD9TmyfzpIcZ6yWW6VU5eOrz+lAGjH96gaoV+Cu/9o9Z9nzKl2SCNUt
3mni1/yJIOoq0N3Yvu8Kq9qHHXWGhaaVEBuI802B+SGizGretMR7AMgVMGYNHp3d
APS2/baaP2P29w7tMwVE1IwnMBVVnfFAV62U+dbhxuqzFJP1bgpngUYVg1Vv
-----END CERTIFICATE-----
[root@K8s-harbor01 certs]#md5sum K8s-harbor01.mooreyxia.com.crt K8s-harbor01.mooreyxia.com.cert
2844fcbe5865448c874255fa5dd43c7f K8s-harbor01.mooreyxia.com.crt
2844fcbe5865448c874255fa5dd43c7f K8s-harbor01.mooreyxia.com.cert
#发现只是文件夹后缀名不同，所以可以简化为复制形式的证书生成
cp -a K8s-harbor01.mooreyxia.com.crt K8s-harbor01.mooreyxia.com.cert- 配置 Docker 客户端使用证书文件
#在docker客户端使用上面的证书文件
#客户端以域名形式建立文件夹
[root@K8s-noded01 ~]#mkdir -pv /etc/docker/certs.d/K8s-harbor01.mooreyxia.com/
mkdir: created directory '/etc/docker/certs.d'
mkdir: created directory '/etc/docker/certs.d/K8s-harbor01.mooreyxia.com/'
#复制证书到客户端
[root@K8s-harbor01 certs]#scp K8s-harbor01.mooreyxia.com.key 192.168.11.214:/etc/docker/certs.d/K8s-harbor01.mooreyxia.com/
[root@K8s-harbor01 certs]#scp K8s-harbor01.mooreyxia.com.cert 192.168.11.214:/etc/docker/certs.d/K8s-harbor01.mooreyxia.com/
[root@K8s-harbor01 certs]#scp ca.crt 192.168.11.214:/etc/docker/certs.d/K8s-harbor01.mooreyxia.com/
#确认证书 - 证书配置无需重启服务即生效
[root@K8s-noded01 ~]#cd /etc/docker/certs.d/K8s-harbor01.mooreyxia.com/
[root@K8s-noded01 K8s-harbor01.mooreyxia.com]#ls
K8s-harbor01.mooreyxia.com.cert K8s-harbor01.mooreyxia.com.key ca.crt
[root@K8s-noded01 certs.d]#tree .
.
├── K8s-harbor01.mooreyxia.com
│ ├── K8s-harbor01.mooreyxia.com.cert
│ ├── K8s-harbor01.mooreyxia.com.key
│ └── ca.crt
└── K8s-harbor02.mooreyxia.com
├── K8s-harbor02.mooreyxia.com.key
└── ca.crt
2 directories, 5 files
#测试登录
[root@K8s-noded01 ~]#docker login K8s-harbor01.mooreyxia.com
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded- Docker 客户端测试推送和拉取镜像
#本地镜像
[root@K8s-noded01 ~]#docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
registry.k8s.io/pause 3.6 6270bb605e12 18 months ago 683kB
#标签生成新镜像
[root@K8s-noded01 ~]#docker tag registry.k8s.io/pause:3.6 K8s-harbor01.mooreyxia.com/library/kubeadmin/registry.k8s.io/pause:3.6
[root@K8s-noded01 ~]#docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
K8s-harbor01.mooreyxia.com/library/kubeadmin/registry.k8s.io/pause 3.6 6270bb605e12 18 months ago 683kB
registry.k8s.io/pause 3.6 6270bb605e12 18 months ago 683kB
#推送镜像到harbor
[root@K8s-noded01 ~]#docker push K8s-harbor01.mooreyxia.com/library/kubeadmin/registry.k8s.io/pause:3.6
The push refers to repository [K8s-harbor01.mooreyxia.com/library/kubeadmin/registry.k8s.io/pause]
1021ef88c797: Pushed
3.6: digest: sha256:74bf6fc6be13c4ec53a86a5acf9fdbc6787b176db0693659ad6ac89f115e182c size: 526- 验证推送是否成功
- 观察是否可以双高同步
部署Haproxy+KeepAlived实现高可用负载均衡
二进制部署Haproxy
#提前下载好需要用的包
[root@K8s-haproxy01 ~]#ls
haproxy-2.7.1.tar.gz lua-5.4.4.tar.gz
#二进制安装脚本
[root@K8s-haproxy01 ~]#cat install_haproxy.sh
#!/bin/bash
HAPROXY_VERSION=2.7.1
HAPROXY_INSTALL_DIR=/apps/haproxy
STATS_AUTH_USER=admin
STATS_AUTH_PASSWORD=123456
HAPROXY_FILE=haproxy-${HAPROXY_VERSION}.tar.gz
LUA_VERSION=5.4.4
LUA_FILE=lua-${LUA_VERSION}.tar.gz
SRC_DIR=/usr/local/src
CWD=`pwd`
CPUS=`lscpu |awk '/^CPU(s)/{print $2}'`
LOCAL_IP=$(hostname -I|awk '{print $1}')
VIP=192.168.10.100
MASTER1=192.168.10.101
MASTER2=192.168.10.102
MASTER3=192.168.10.103
. /etc/os-release
color () {
RES_COL=60
MOVE_TO_COL="echo -en \033[${RES_COL}G"
SETCOLOR_SUCCESS="echo -en \033[1;32m"
SETCOLOR_FAILURE="echo -en \033[1;31m"
SETCOLOR_WARNING="echo -en \033[1;33m"
SETCOLOR_NORMAL="echo -en E[0m"
echo -n "$1" && $MOVE_TO_COL
echo -n "["
if [ $2 = "success" -o $2 = "0" ] ;then
${SETCOLOR_SUCCESS}
echo -n $" OK "
elif [ $2 = "failure" -o $2 = "1" ] ;then
${SETCOLOR_FAILURE}
echo -n $"FAILED"
else
${SETCOLOR_WARNING}
echo -n $"WARNING"
fi
${SETCOLOR_NORMAL}
echo -n "]"
echo
}
check_lua_file (){
if [ ! -e ${LUA_FILE} ];then
color "缺少${LUA_FILE}文件!" 1
exit
else
color "相关文件已准备!" 0
fi
}
check_haproxy_file (){
if [ ! -e ${HAPROXY_FILE} ];then
color "缺少${HAPROXY_FILE}文件!" 1
exit
else
color "相关文件已准备!" 0
fi
}
install_packs () {
if [ $ID = "centos" -o $ID = "rocky" ];then
yum -y install gcc make gcc-c++ glibc glibc-devel pcre pcre-devel openssl openssl-devel systemd-devel libtermcap-devel ncurses-devel libevent-devel readline-devel
elif [ $ID = "ubuntu" ];then
apt update
apt -y install gcc make openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev libreadline-dev libsystemd-dev liblua5.3-dev
else
color "不支持此操作系统!" 1
fi
[ $? -eq 0 ] || { color '安装软件包失败,退出!' 1; exit; }
}
install_lua () {
cd ${CWD}
check_lua_file
tar xf ${LUA_FILE} -C ${SRC_DIR}
LUA_DIR=${LUA_FILE%.tar*}
cd ${SRC_DIR}/${LUA_DIR}
make all test
}
install_haproxy(){
check_haproxy_file
cd ${CWD}
tar xf ${HAPROXY_FILE} -C ${SRC_DIR}
HAPROXY_DIR=${HAPROXY_FILE%.tar*}
cd ${SRC_DIR}/${HAPROXY_DIR}
if [[ ${VERSION_ID} =~ ^(7|8) ]] ;then
install_lua
cd ${SRC_DIR}/${HAPROXY_DIR}
make -j ${CPUS} ARCH=x86_64 TARGET=linux-glibc USE_PROMEX=1 USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 USE_SYSTEMD=1 USE_CPU_AFFINITY=1 USE_LUA=1 LUA_INC=${SRC_DIR}/${LUA_DIR}/src/ LUA_LIB=${SRC_DIR}/${LUA_DIR}/src/ PREFIX=${HAPROXY_INSTALL_DIR}
else
make -j ${CPUS} ARCH=x86_64 TARGET=linux-glibc USE_PROMEX=1 USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 USE_SYSTEMD=1 USE_CPU_AFFINITY=1 USE_LUA=1
fi
make install PREFIX=${HAPROXY_INSTALL_DIR}
[ $? -eq 0 ] && color "HAPROXY编译安装成功" 0 || { color "HAPROXY编译安装失败,退出!" 1;exit; }
[ -L /usr/sbin/haproxy ] || ln -s ${HAPROXY_INSTALL_DIR}/sbin/haproxy /usr/sbin/
[ -d /etc/haproxy ] || mkdir /etc/haproxy
[ -d /var/lib/haproxy/ ] || mkdir -p /var/lib/haproxy/
cat > /etc/haproxy/haproxy.cfg /lib/systemd/system/haproxy.service /dev/null && color 'HAPROXY安装完成!' 0 || { color 'HAPROXY 启动失败,退出!' 1; exit; }
echo "-------------------------------------------------------------------"
echo -e "请访问链接: E[32;1mhttp://${LOCAL_IP}:9999/haproxy-statusE[0m"
echo -e "用户和密码: E[32;1m${STATS_AUTH_USER}/${STATS_AUTH_PASSWORD}E[0m"
}
install_packs
install_haproxy
start_haproxy
[root@K8s-haproxy01 ~]#bash install_haproxy.sh
#查看版本
[root@K8s-haproxy01 ~]#haproxy -v
HAProxy version 2.7.1-3e4af0e 2022/12/19 - https://haproxy.org/
Status: stable branch - will stop receiving fixes around Q1 2024.
Known bugs: http://www.haproxy.org/bugs/bugs-2.7.1.html
Running on: Linux 5.15.0-60-generic #66-Ubuntu SMP Fri Jan 20 14:29:49 UTC 2023 x86_64
#以Service形式管理进程
[root@K8s-haproxy01 ~]#cat /lib/systemd/system/haproxy.service
[Unit]
Descriptinotallow=HAProxy Load Balancer
After=syslog.target network.target
[Service]
ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid
ExecReload=/bin/kill -USR2
[Install]
WantedBy=multi-user.target
[root@K8s-haproxy01 ~]#systemctl status haproxy
● haproxy.service - HAProxy Load Balancer
Loaded: loaded (/lib/systemd/system/haproxy.service; enabled; vendor preset: enabled)
Active: active (running) since Sat 2023-03-18 13:01:09 UTC; 5min ago
Main PID: 2712 (haproxy)
Tasks: 3 (limit: 2237)
Memory: 17.6M
CPU: 220ms
CGroup: /system.slice/haproxy.service
├─2712 /usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid
└─2716 /usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /var/lib/haproxy/haproxy.pid
Mar 18 13:01:09 K8s-haproxy01.mooreyxia.com systemd[1]: Starting HAProxy Load Balancer...
Mar 18 13:01:09 K8s-haproxy01.mooreyxia.com systemd[1]: Started HAProxy Load Balancer.
Mar 18 13:01:09 K8s-haproxy01.mooreyxia.com haproxy[2712]: [NOTICE] (2712) : haproxy version is 2.7.1-3e4af0e
Mar 18 13:01:09 K8s-haproxy01.mooreyxia.com haproxy[2712]: [NOTICE] (2712) : path to executable is /usr/sbin/haproxy
Mar 18 13:01:09 K8s-haproxy01.mooreyxia.com haproxy[2712]: [ALERT] (2712) : config : parsing [/etc/haproxy/haproxy.cfg:8] : 'pidfile' already specified. Continuing.
Mar 18 13:01:09 K8s-haproxy01.mooreyxia.com haproxy[2712]: [NOTICE] (2712) : New worker (2716) forked
Mar 18 13:01:09 K8s-haproxy01.mooreyxia.com haproxy[2712]: [NOTICE] (2712) : Loading success.
二进制部署KeepAlived
#下载最新版安装包
[root@K8s-haproxy01 ~]#ls
keepalived-2.2.7.tar.gz
#二进制安装脚本
[root@K8s-haproxy01 ~]#cat install_keepalived.sh
#!/bin/bash
KEEPALIVED_VERSION=2.2.7
KEEPALIVED_FILE=keepalived-${KEEPALIVED_VERSION}.tar.gz
KEEPALIVED_INSTALL_DIR=/apps/keepalived
SRC_DIR=/usr/local/src
KEEPALIVED_URL=https://keepalived.org/software/
CPUS=`grep -c processor /proc/cpuinfo`
. /etc/os-release
color () {
RES_COL=60
MOVE_TO_COL="echo -en \033[${RES_COL}G"
SETCOLOR_SUCCESS="echo -en \033[1;32m"
SETCOLOR_FAILURE="echo -en \033[1;31m"
SETCOLOR_WARNING="echo -en \033[1;33m"
SETCOLOR_NORMAL="echo -en E[0m"
echo -n "$1" && $MOVE_TO_COL
echo -n "["
if [ $2 = "success" -o $2 = "0" ] ;then
${SETCOLOR_SUCCESS}
echo -n $" OK "
elif [ $2 = "failure" -o $2 = "1" ] ;then
${SETCOLOR_FAILURE}
echo -n $"FAILED"
else
${SETCOLOR_WARNING}
echo -n $"WARNING"
fi
${SETCOLOR_NORMAL}
echo -n "]"
echo
}
download_file (){
cd ${SRC_DIR}
if [ $ID = 'centos' -o $ID = 'rocky' ];then
rpm -q wget &> /dev/null || yum -y install wget
elif [ $ID = 'ubuntu' ];then
dpkg -l |grep wget || { apt update; apt install -y wget; }
else
color "不支持此操作系统，退出!" 1
exit
fi
if [ ! -e ${KEEPALIVED_FILE} ];then
wget --no-check-certificate ${KEEPALIVED_URL}${KEEPALIVED_FILE}
[ $? -ne 0 ] && { color "KEEPALIVED源码包下载失败" 1 ; exit; }
fi
}
install_keepalived () {
if [ $ID = 'centos' -o $ID = 'rocky' ];then
yum -y install make gcc ipvsadm autoconf automake openssl-devel libnl3-devel iptables-devel net-snmp-devel glib2-devel pcre2-devel libmnl-devel systemd-devel &> /dev/null
elif [ $ID = 'ubuntu' ];then
apt update
apt -y install make gcc ipvsadm build-essential pkg-config automake autoconf libipset-dev libnl-3-dev libnl-genl-3-dev libssl-dev libxtables-dev libip4tc-dev libip6tc-dev libipset-dev libmagic-dev libsnmp-dev libglib2.0-dev libpcre2-dev libnftnl-dev libmnl-dev libsystemd-dev
else
color "不支持此操作系统，退出!" 1
fi
tar xf ${KEEPALIVED_FILE}
cd keepalived-${KEEPALIVED_VERSION}
./configure --prefix=${KEEPALIVED_INSTALL_DIR} --disable-fwmark
make -j $CPUS && make install
if [ $? -eq 0 ];then
color "KEEPALIVED编译安装成功" 0
else
color "KEEPALIVED编译安装失败,退出!" 1
exit
fi
[ -d /etc/keepalived ] || mkdir -p /etc/keepalived
cp ${KEEPALIVED_INSTALL_DIR}/etc/keepalived/keepalived.conf.sample /etc/keepalived/keepalived.conf
cp ./keepalived/keepalived.service /lib/systemd/system/
}
start_keepalived () {
systemctl daemon-reload
systemctl enable --now keepalived &> /dev/null
systemctl is-active keepalived
if [ $? -eq 0 ] ;then
color "Keepalived 服务安装成功!" 0
else
color "Keepalived 服务安装失败!" 1
exit 1
fi
}
download_file
install_keepalived
start_keepalived
[root@K8s-haproxy01 ~]#bash install_keepalived.sh
#确认服务运行成功
[root@K8s-haproxy01 ~]#systemctl status keepalived.service
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/lib/systemd/system/keepalived.service; enabled; vendor preset: enabled)
Active: active (running) since Sat 2023-03-18 13:29:42 UTC; 2min 20s ago
Docs: man:keepalived(8)
man:keepalived.conf(5)
man:genhash(1)
https://keepalived.org
Main PID: 43092 (keepalived)
Tasks: 3 (limit: 2237)
Memory: 3.7M
CPU: 96ms
CGroup: /system.slice/keepalived.service
├─43092 /apps/keepalived/sbin/keepalived --dont-fork -D
├─43093 /apps/keepalived/sbin/keepalived --dont-fork -D
└─43094 /apps/keepalived/sbin/keepalived --dont-fork -D
Mar 18 13:29:51 K8s-haproxy01.mooreyxia.com Keepalived_healthcheckers[43093]: HTTP_CHECK on service [192.168.200.2]:tcp:1358 failed after 3 retries.
Mar 18 13:29:51 K8s-haproxy01.mooreyxia.com Keepalived_healthcheckers[43093]: Removing service [192.168.200.2]:tcp:1358 from VS [10.10.10.2]:tcp:1358
Mar 18 13:29:51 K8s-haproxy01.mooreyxia.com Keepalived_healthcheckers[43093]: Lost quorum 1-0=1 > 0 for VS [10.10.10.2]:tcp:1358
Mar 18 13:29:51 K8s-haproxy01.mooreyxia.com Keepalived_healthcheckers[43093]: Adding sorry server [192.168.200.200]:tcp:1358 to VS [10.10.10.2]:tcp:1358
Mar 18 13:29:51 K8s-haproxy01.mooreyxia.com Keepalived_healthcheckers[43093]: Removing alive servers from the pool for VS [10.10.10.2]:tcp:1358
Mar 18 13:30:17 K8s-haproxy01.mooreyxia.com Keepalived_healthcheckers[43093]: Timeout opening connection to remote SMTP server [192.168.200.1]:25.
Mar 18 13:30:18 K8s-haproxy01.mooreyxia.com Keepalived_healthcheckers[43093]: Timeout opening connection to remote SMTP server [192.168.200.1]:25.
Mar 18 13:30:18 K8s-haproxy01.mooreyxia.com Keepalived_healthcheckers[43093]: Timeout opening connection to remote SMTP server [192.168.200.1]:25.
Mar 18 13:30:19 K8s-haproxy01.mooreyxia.com Keepalived_healthcheckers[43093]: Timeout opening connection to remote SMTP server [192.168.200.1]:25.
Mar 18 13:30:21 K8s-haproxy01.mooreyxia.com Keepalived_healthcheckers[43093]: Timeout opening connection to remote SMTP server [192.168.200.1]:25.更改KeepAlived配置文件实现VRRP
- 实现master/slave的 Keepalived 单主架构
#MASTER配置
[root@K8s-haproxy01 keepalived]#vim keepalived.conf
[root@K8s-haproxy01 keepalived]#cat keepalived.conf
global_defs {
router_id K8s-haproxy01.mooreyxia.com #每个keepalived主机唯一标识，建议使用当前主机名
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
vrrp_mcast_group4 224.1.1.1 #默认组播IP地址，可指定组播范围：224.0.0.0到239.255.255.255
}
vrrp_instance VI_1 { #vrrp的实例名,一般为业务名称
state MASTER #当前节点在此虚拟路由器上的初始状态，状态为MASTER或者BACKUP
interface eth1 #绑定为当前VRRP虚拟路由器使用的物理接口做集群内心跳检测.注意：如果和虚拟IP一样，有脑裂现象
virtual_router_id 88 #每个虚拟路由器唯一标识，同一集群内公用
priority 100 #当前物理节点在此虚拟路由器的优先级
advert_int 1 #vrrp通告的时间间隔，默认1s
authentication { #认证机制
auth_type PASS #AH为IPSEC认证(不推荐),PASS为简单密码(建议使用)
auth_pass 123456 #预共享密钥，仅前8位有效，同一个虚拟路由器的多个keepalived节点必须一样
}
virtual_ipaddress { #指定VIP的网卡，建议和interface指令指定的网卡不在一个网卡
192.168.11.241/24 dev eth0 lable eth0:1
192.168.11.242/24 dev eth0 lable eth0:1
192.168.11.243/24 dev eth0 lable eth0:1
192.168.11.244/24 dev eth0 lable eth0:1
192.168.11.245/24 dev eth0 lable eth0:1
}
}
[root@K8s-haproxy01 ~]#systemctl restart keepalived.service
[root@K8s-haproxy01 ~]#hostname -I
192.168.11.203 192.168.11.241 192.168.11.242 192.168.11.243 192.168.11.244 192.168.11.245 192.168.11.246
#BACKUP配置
[root@K8s-haproxy02 ~]#cat /etc/keepalived/keepalived.conf
global_defs {
router_id K8s-haproxy02.mooreyxia.com
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
vrrp_mcast_group4 224.1.1.1
}
vrrp_instance VI_1 {
state BACKUP
interface eth1
virtual_router_id 88
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass 123456
}
virtual_ipaddress {
192.168.11.241/24 dev eth0 lable eth0:1
192.168.11.242/24 dev eth0 lable eth0:1
192.168.11.243/24 dev eth0 lable eth0:1
192.168.11.244/24 dev eth0 lable eth0:1
192.168.11.245/24 dev eth0 lable eth0:1
}
}
[root@K8s-haproxy02 ~]#systemctl restart keepalived.service
[root@K8s-haproxy02 ~]#hostname -I
192.168.11.204 192.168.11.247 - 测试KeepAlived是否会漂移IP
#观察发现此时VIP都挂载在haproxy01服务器上
[root@K8s-haproxy01 ~]#hostname -I
192.168.11.203 192.168.11.241 192.168.11.242 192.168.11.243 192.168.11.244 192.168.11.245 192.168.11.246
#通过停止keepalived服务观察心跳检测prio权重变化
[root@K8s-haproxy01 ~]#tcpdump -i eth1 -nn host 224.1.1.1
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on eth1, link-type EN10MB (Ethernet), snapshot length 262144 bytes
04:34:12.768633 IP 192.168.11.8 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 36
04:34:13.768816 IP 192.168.11.8 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 36
04:34:18.769724 IP 192.168.11.8 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 36
04:34:19.410563 IP 192.168.11.8 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 0, authtype simple, intvl 1s, length 36
04:34:20.098902 IP 192.168.11.11 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 36
04:34:30.104946 IP 192.168.11.11 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 36
04:34:31.105208 IP 192.168.11.11 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 36
04:34:31.431327 IP 192.168.11.8 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 36
04:34:32.436616 IP 192.168.11.8 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 36
04:34:33.436796 IP 192.168.11.8 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 36
04:34:34.436981 IP 192.168.11.8 > 224.1.1.1: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 36
更改Haproxy配置文件实现负载均衡
- 监听至kubernetes（在Kubernetes部署集群部署完成后添加反向代理即可，未搭载完成此处先跳过）
[root@K8s-haproxy01 ~]#cat /etc/haproxy/haproxy.cfg
global
maxconn 100000
stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
uid 99
gid 99
daemon
pidfile /var/lib/haproxy/haproxy.pid
log 127.0.0.1 local3 info
defaults
option http-keep-alive
option forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client 300000ms
timeout server 300000ms
listen stats
mode http
bind 0.0.0.0:9999
stats enable
log global
stats uri /haproxy-status
stats auth admin:123456
#反向代理Kubernetes的管理面板
#listen kubernetes-6443
# bind 192.168.10.100:6443
# mode tcp
# log global
# server 192.168.10.101 192.168.10.101:6443 check inter 3000 fall 2 rise 5
# server 192.168.10.102 192.168.10.102:6443 check inter 3000 fall 2 rise 5
# server 192.168.10.103 192.168.10.102:6443 check inter 3000 fall 2 rise 5
我是moore,大家一起加油！！！
服务器托管，北京服务器托管，服务器租用 http://www.fwqtg.net
机房租用，北京机房租用，IDC机房托管， http://www.e1idc.net
