17、ELK
helm 安装 elkfk（kafka 集群外可访问）
ES/Kibana
部署顺序：
1、elasticsearch
2、kibana
3、kafka
4、logstash
5、filebeat
kubectl create ns elk
helm3部署elkfk
1、elasticsearch
helm repo add elastic https://helm.elastic.co
helm repo list
helm repo update
helm search repo elastic/elasticsearch
cd && helm pull elastic/elasticsearch --untar --version 7.17.3
cd elasticsearch
cat > values-prod.yaml EOF
# 集群名称
clusterName: "elasticsearch"
# ElasticSearch 6.8+ 默认安装了 x-pack 插件，部分功能免费，这里选禁用
image: "docker.elastic.co/elasticsearch/elasticsearch"
imageTag: "7.17.3"
imagePullPolicy: "IfNotPresent"
esConfig:
elasticsearch.yml: |
network.host: 0.0.0.0
cluster.name: "elasticsearch"
xpack.security.enabled: false
resources:
limits:
cpu: "2"
memory: "4Gi"
requests:
cpu: "1"
memory: "2Gi"
volumeClaimTemplate:
storageClassName: "nfs-storage"
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 2Ti
service:
type: NodePort
port: 9000
nodePort: 31311
EOF
禁用 Kibana 安全提示（Elasticsearch built-in security features are not enabled）xpack.security.enabled: false
helm upgrade --install --namespace elk es -f ./values-prod.yaml .
验证
curl 192.168.1.200:31311/_cat/health
curl 192.168.1.200:31311/_cat/nodes
2、kibana
helm search repo elastic/kibana
cd && helm pull elastic/kibana --untar --version 7.17.3
cd kibana
cat > values-prod.yaml 'EOF'
kibanaConfig:
kibana.yml: |
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.hosts: [ "http://elasticsearch-master-headless:9200" ]
resources:
limits:
cpu: "2"
memory: "2Gi"
requests:
cpu: "1"
memory: "1Gi"
kibanaConfig:
kibana.yml: |
i18n.locale: "zh-CN"
service:
#type: ClusterIP
type: NodePort
loadBalancerIP: ""
port: 5601
nodePort: "30026"
EOF
helm upgrade --install --namespace elk kibana -f ./values-prod.yaml .
cat > ~/kibana/kibana-Ingress.yml 'EOF'
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: kibana-ingress
namespace: elk
annotations:
nginx.ingress.kubernetes.io/ssl-redirect: 'true'
nginx.ingress.kubernetes.io/proxy-body-size: '4G'
nginx.ingress.kubernetes.io/auth-type: basic
nginx.ingress.kubernetes.io/auth-secret: kibana-auth-secret
nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - admin'
spec:
ingressClassName: nginx
rules:
- host: kibana.huanghuanhui.cloud
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: kibana-kibana
port:
number: 5601
tls:
- hosts:
- kibana.huanghuanhui.cloud
secretName: kibana-ingress-tls
EOF
yum -y install httpd-tools
cd ~/kibana && htpasswd -bc auth admin Admin@2024
kubectl create secret generic kibana-auth-secret --from-file=auth -n elk
kubectl create secret -n elk
tls kibana-ingress-tls
--key=/root/ssl/huanghuanhui.cloud.key
--cert=/root/ssl/huanghuanhui.cloud.crt
kubectl apply -f ~/kibana/kibana-Ingress.yml
访问地址：kibana.huanghuanhui.cloud
账号密码：admin、Admin@2024
http://192.168.1.201:30026/app/dev_tools#/console
GET _cat/nodes
GET _cat/health
GET _cat/indices
3、kafka（k8s部署kafka集群 ==》外部访问）
mkdir -p ~/kafka-yml && cd ~/kafka-yml
cat > ~/kafka-yml/zk.yml 'EOF'
apiVersion: v1
kind: Service
metadata:
labels:
app: zookeeper-cluster
namespace: elk
name: zookeeper-cluster
spec:
selector:
app: zookeeper-cluster
ports:
- name: client
port: 2181
targetPort: 2181
- name: follower
port: 2888
targetPort: 2888
- name: leader
port: 3888
targetPort: 3888
clusterIP: None
---
apiVersion: v1
kind: Service
metadata:
namespace: elk
name: zookeeper-cs
spec:
selector:
app: zookeeper-cluster
type: NodePort
ports:
- name: client
port: 2181
nodePort: 30152
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
namespace: elk
name: crs-zookeeper
spec:
replicas: 3
podManagementPolicy: Parallel
serviceName: zookeeper-cluster
selector:
matchLabels:
app: zookeeper-cluster
template:
metadata:
labels:
component: zookeeper-cluster
app: zookeeper-cluster
spec:
containers:
- name: zookeeper
image: bitnami/zookeeper:3.8.2
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 0
ports:
- containerPort: 2181
- containerPort: 2888
- containerPort: 3888
lifecycle:
postStart:
exec:
command:
- "sh"
- "-c"
- >
echo $(( $(cat /etc/hosts | grep zookeeper | awk '{print($3)}' | awk '{split($0,array,"-")} END{print array[3]}') + 1 )) > /bitnami/zookeeper/data/myid
env:
- name: ALLOW_ANONYMOUS_LOGIN
value: "yes"
- name: ZOO_SERVERS
value: crs-zookeeper-0.zookeeper-cluster.elk.svc.cluster.local:2888:3888,crs-zookeeper-1.zookeeper-cluster.elk.svc.cluster.local:2888:3888,crs-zookeeper-2.zookeeper-cluster.elk.svc.cluster.local:2888:3888
volumeMounts:
- name: zoodata-outer
mountPath: /bitnami/zookeeper
volumeClaimTemplates:
- metadata:
name: zoodata-outer
spec:
storageClassName: nfs-storage
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: 2Ti
EOF
kubectl apply -f ~/kafka-yml/zk.yml
cat > ~/kafka-yml/kafka.yml 'EOF'
apiVersion: v1
kind: Service
metadata:
namespace: elk
name: kafka-headless
spec:
selector:
app: kafka-cluster
ports:
- name: client
port: 9092
targetPort: 9092
clusterIP: None
---
apiVersion: v1
kind: Service
metadata:
name: kafka-0
namespace: elk
labels:
app: kafka-cluster
spec:
ports:
- port: 9092
targetPort: 9092
nodePort: 30127
name: server
type: NodePort
selector:
statefulset.kubernetes.io/pod-name: crs-kafka-0
# app: kafka-cluster
---
apiVersion: v1
kind: Service
metadata:
name: kafka-1
namespace: elk
labels:
app: kafka-cluster
spec:
ports:
- port: 9092
targetPort: 9092
nodePort: 30128
name: server
type: NodePort
selector:
statefulset.kubernetes.io/pod-name: crs-kafka-1
---
apiVersion: v1
kind: Service
metadata:
name: kafka-2
namespace: elk
labels:
app: kafka-cluster
spec:
ports:
- port: 9092
targetPort: 9092
nodePort: 30129
name: server
type: NodePort
selector:
statefulset.kubernetes.io/pod-name: crs-kafka-2
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
namespace: elk
name: crs-kafka
spec:
replicas: 3
podManagementPolicy: Parallel
serviceName: kafka-cluster
selector:
matchLabels:
app: kafka-cluster
template:
metadata:
labels:
app: kafka-cluster
spec:
hostname: kafka
containers:
- name: kafka
command:
- bash
- -ec
- |
HOSTNAME=`hostname -s`
if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then
ORD=${BASH_REMATCH[2]}
PORT=$((ORD + 30127))
export KAFKA_CFG_ADVERTISED_LISTENERS="PLAINTEXT://192.168.1.200:$PORT"
else
echo "Failed to get index from hostname $HOST"
exit 1
fi
exec /entrypoint.sh /run.sh
image: bitnami/kafka:3.5.1
# image: bitnami/kafka:latest
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 0
# resources:
# requests:
# memory: "1G"
# cpu: "0.5"
ports:
- containerPort: 9092
env:
- name: KAFKA_CFG_ZOOKEEPER_CONNECT
value: crs-zookeeper-0.zookeeper-cluster.elk.svc.cluster.local:2181,crs-zookeeper-1.zookeeper-cluster.elk.svc.cluster.local:2181,crs-zookeeper-2.zookeeper-cluster.elk.svc.cluster.local:2181
# value: zookeeper-cluster:2181
- name: ALLOW_PLAINTEXT_LISTENER
value: "yes"
volumeMounts:
- name: kafkadata-outer
mountPath: /bitnami/kafka
volumeClaimTemplates:
- metadata:
name: kafkadata-outer
spec:
storageClassName: nfs-storage
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: 2Ti
EOF
kubectl apply -f ~/kafka-yml/kafka.yml
注意修改yml文件98行里面的export的ip地址
这里修改为公网的ip：58.34.61.154（内网192.168.1.200）
kafka ui
docker pull provectuslabs/kafka-ui:latest
docker pull freakchicken/kafka-ui-lite
docker run -d
--name kafka-ui1
--restart always
--privileged=true
-p 8888:8080
-e KAFKA_CLUSTERS_0_NAME=k8s-kafka
-e KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=192.168.1.200:30127,192.168.1.200:30128,192.168.1.200:30129
provectuslabs/kafka-ui:latest
访问地址：192.168.1.200:8888
docker run -d
--name kafka-ui2
--restart always
--privileged=true
-p 8889:8889
freakchicken/kafka-ui-lite
访问地址：192.168.1.200:8889
4、filebeat
1、k8s方式
helm search repo elastic/filebeat
cd && 服务器托管网helm pull elastic/filebeat --untar --version 7.17.3
cd filebeat
cat > values-prod.yaml 'EOF'
daemonset:
filebeatConfig:
filebeat.yml: |
filebeat.inputs:
- type: container
paths:
- /var/log/containers/*.log
output.elasticsearch:
enabled: false
host: '${NODE_NAME}'
hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}'
output.kafka:
enabled: true
hosts: ["192.168.1.200:30127","192.168.1.200:30128","192.168.1.200:30129"]
topic: k8s-logs
EOF
helm upgrade --install --namespace elk filebeat -f ./values-prod.yaml .
2、docker方式
cat > filebeat.yml 'EOF'
# 日志输入配置（可配置多个）
filebeat.inputs:
- type: log
enabled: true
paths:
- /mnt/nfs/logs/*/*.log
tags: ["dev-c"]
fields:
server: dev-c
fields_under_root: true
#日志输出配置
output.kafka:
enabled: true
hosts: ["192.168.1.200:30127","192.168.1.200:30128","192.168.1.200:30129"]
topic: "dev-c"
partition.round_robin:
reachable_only: false
required_acks: 1
compression: gzip
max_message_bytes: 1000000
EOF
docker run -d --name filebeat
--user=root
--restart=always
-v /mnt/nfs/logs/:/mnt/nfs/logs/
-v /root/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml
-v /etc/localtime:/etc/localtime
-v /etc/timezone:/etc/timezone
elastic/filebeat:7.17.3
5、logstash
helm search repo elastic/logstash
cd && helm pull elastic/logstash --untar --version 7.17.3
cd logstash
cat > values-prod.yaml 'EOF'
logstashConfig:
logstash.yml: |
xpack.monitoring.enabled: false
logstashPipeline:
logstash.yml: |
input {
kafka {
bootstrap_servers => "192.168.1.200:30127,192.168.1.200:30128,192.168.1.200:30129"
topics => ["k8s-logs"]
#group_id => "mygroup"
#如果使用元数据就不能使用下面的byte字节序列化，否则会报错
#key_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
#value_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
consumer_threads => 1
#默认为false，只有为true的时候才会获取到元数据
decorate_events => true
auto_offset_reset => "earliest"
}
}
filter {
mutate {
#从kafka的key中获取数据并按照逗号切割
split => ["[@metadata][kafka][key]", ","]
add_field => {
#将切割后的第一位数据放入自定义的“index”字段中
"index" => "%{[@metadata][kafka][key][0]}"
}
}
}
output {
elasticsearch {
pool_max => 1000
pool_max_per_route => 200
hosts => ["elasticsearch-master-headless.elk.svc.cluster.local:9200"]
index => "k8s-logs-%{+YYYY.MM.dd}"
}
}
# 资源限制
resources:
requests:
cpu: "100m"
memory: "256Mi"
limits:
cpu: "1000m"
memory: "1Gi"
persistence:
enabled: true
volumeClaimTemplate:
accessModes: ["ReadWriteOnce"]
storageClassName: nfs-storage
resources:
requests:
storage: 2Ti
EOF
helm upgrade --install --namespace elk logstash -f ./values-prod.yaml .
手撕yml
mkdir -p ~/logstash-yml && cd ~/logstash-yml
cat > logstash.yaml 'EOF'
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-dev-configmap
namespace: elk
data:
logstash.yml: |
http.host: "0.0.0.0"
path.config: /usr/share/logstash/pipeline
logstash.conf: |
input {
kafka {
bootstrap_servers => "192.168.1.200:30127,192.168.1.200:30128,192.168.1.200:30129"
topics => ["dev"]
codec => "json"
type => "dev"
group_id => "dev"
consumer_threads => 1
}
}
filter {
if [type] == "dev" {
json {
source => ["message"]
remove_field => ["offset","host","beat","@version","event","agent","ecs"]
}
mutate {
add_field => {
project_path => "%{[log][file][path]}"
}
}
mutate {
split => ["project_path", "/"]
add_field => {
"project_name" => "%{[project_path][-3]}"
}
}
date {
match => ["time","yyyy-MM-dd HH:mm:ss.SSS"]
timezone => "Asia/Shanghai"
target => "@timestamp"
}
mutate {
remove_field => ["log","project_path","time","input"]
}
}
}
output {
elasticsearch {
hosts => ["elasticsearch-master-headless.elk.svc.cluster.local:9200"]
index => "dev-%{+YYYY.MM.dd}"
}
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logstash-dev
namespace: elk
spec:
selector:
服务器托管网 matchLabels:
app: logstash-dev
replicas: 1
template:
metadata:
labels:
app: logstash-dev
spec:
containers:
- name: logstash-dev
image: docker.elastic.co/logstash/logstash:7.17.3
ports:
- containerPort: 5044
volumeMounts:
- name: logstash-pipeline-volume
mountPath: /usr/share/logstash/pipeline
- mountPath: /etc/localtime
name: localtime
volumes:
- name: logstash-pipeline-volume
configMap:
name: logstash-dev-configmap
items:
- key: logstash.conf
path: logstash.conf
- hostPath:
path: /etc/localtime
name: localtime
---
kind: Service
apiVersion: v1
metadata:
name: logstash-dev
namespace: elk
spec:
selector:
app: logstash
type: ClusterIP
ports:
- protocol: TCP
port: 5044
targetPort: 5044
EOF
kubectl apply -f logstash.yaml
服务器托管，北京服务器托管，服务器租用 http://www.fwqtg.net
