ELK 8.6.2 +filebeat docker配置（ssl认证）

企业开发 2023-06-18 23:50:24 阅读次数: 0

filebeat —> logstash —>elasticsearch —> kibana

官网地址：

https://www.elastic.co/guide/en/kibana/current/settings.html
https://www.elastic.co/guide/en/logstash/current/docker-config.html
https://www.elastic.co/guide/en/elasticsearch/reference/8.6/docker.html

一、centos7 yum安装源

yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

二、docker部署

ELK和Filebeat组合在一起可以实现更强大的日志收集和处理功能。

1. 创建Docker网络(省略)

ELK和Filebeat的容器将会在同一个网络下通信，因此我们需要先创建一个Docker网络。

docker network create  -d overlay --attachable  elastic-net

docker service部署elk3副本(可忽略，笔记)

docker service create --name es --network elastic-net --replicas 3 -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" elasticsearch:8.6.2

docker service logs -f es

curl -X GET "localhost:9200/_cat/nodes?v&pretty"

2. 部署Elasticsearch、kibana容器

我们将在Docker容器中部署Elasticsearch。

在这个命令中，我们使用docker run命令创建并启动一个名为elasticsearch的容器。-p选项将Elasticsearch容器的端口4映射到主机的端口，–network选项将容器连接到之前创建的elk-network网络。最后，我们将discovery.type配置为single-node，以指示该节点是一个独立节点。

a、创建所需目录及yml

mkdir -p /u01/elk/elasticsearch/es0{
    
    1,2,3}/data
mkdir -p /u01/elk/elasticsearch/es0{
    
    1,2,3}/logs
mkdir -p /u01/elk/kibana/{
    
    data,config}
mkdir -p /u01/elk/logstash/{
    
    data,config}
chmod -R 777 /u01/elk/*

a1、编写docker-copose.yml

cd /u01/elk/elasticsearch/
vim docker-compose.yml

version: "2.2"

services:
  setup:
    image: elasticsearch:8.6.2
    volumes:
      - ./certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x${
    
    ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${
    
    KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: es01\n"\
          "    dns:\n"\
          "      - es01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es02\n"\
          "    dns:\n"\
          "      - es02\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          "  - name: es03\n"\
          "    dns:\n"\
          "      - es03\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{
    
    \} \;;
        find . -type f -exec chmod 640 \{
    
    \} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${
    
    ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{
    
    \"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{
    
    }"; do sleep 10; done;
        echo "All done!";
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120

  es01:
    container_name: es01
    depends_on:
      setup:
        condition: service_healthy
    image: elasticsearch:8.6.2
    volumes:
      - ./certs:/usr/share/elasticsearch/config/certs
      - ./es01/data:/usr/share/elasticsearch/data
      - ./es01/logs:/usr/share/elasticsearch/logs
    ports:
      - 9200:9200
      - 9300:9300
    environment:
      - node.name=es01
      - cluster.name=${
    
    CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es02,es03
      - ELASTIC_PASSWORD=${
    
    ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es01/es01.key
      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es01/es01.key
      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${
    
    LICENSE}
    mem_limit: ${
    
    MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    deploy:
      resources:
        limits:
          cpus: '0.5'
          memory: 1g
        reservations:
          cpus: '0.25'
          memory: 512m
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  es02:
    container_name: es02
    depends_on:
      - es01
    image: elasticsearch:8.6.2
    volumes:
      - ./certs:/usr/share/elasticsearch/config/certs
      - ./es02/data:/usr/share/elasticsearch/data
      - ./es02/logs:/usr/share/elasticsearch/logs
    environment:
      - node.name=es02
      - cluster.name=${
    
    CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es03
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es02/es02.key
      - xpack.security.http.ssl.certificate=certs/es02/es02.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es02/es02.key
      - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${
    
    LICENSE}
    mem_limit: ${
    
    MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    deploy:
      resources:
        limits:
          cpus: '0.5'
          memory: 1g
        reservations:
          cpus: '0.25'
          memory: 512m
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  es03:
    container_name: es03
    depends_on:
      - es02
    image: elasticsearch:8.6.2
    volumes:
      - ./certs:/usr/share/elasticsearch/config/certs
      - ./es03/logs:/usr/share/elasticsearch/logs
      - ./es03/data:/usr/share/elasticsearch/data
    environment:
      - node.name=es03
      - cluster.name=${
    
    CLUSTER_NAME}
      - cluster.initial_master_nodes=es01,es02,es03
      - discovery.seed_hosts=es01,es02
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/es03/es03.key
      - xpack.security.http.ssl.certificate=certs/es03/es03.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/es03/es03.key
      - xpack.security.transport.ssl.certificate=certs/es03/es03.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${
    
    LICENSE}
    mem_limit: ${
    
    MEM_LIMIT}
    ulimits:
      memlock:
        soft: -1
        hard: -1
    deploy:
      resources:
        limits:
          cpus: '0.5'
          memory: 1g
        reservations:
          cpus: '0.25'
          memory: 512m
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

  kibana:
    container_name: kibana
    depends_on:
      es01:
        condition: service_healthy
      es02:
        condition: service_healthy
      es03:
        condition: service_healthy
    image: kibana:8.6.2
    volumes:
      - ./certs:/usr/share/kibana/config/certs
      - /u01/elk/kibana/data:/usr/share/kibana/data
    ports:
      - ${
    
    KIBANA_PORT}:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://es01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${
    
    KIBANA_PASSWORD}
      - SERVER_PUBLICBASEURL=http://192.168.174.130:5601
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
    mem_limit: ${
    
    MEM_LIMIT}
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
        ]
      interval: 10s
      timeout: 10s
      retries: 120

#volumes:
#  certs:
#    driver: local
#  esdata01:
#    driver: local
#  esdata02:
#    driver: local
#  esdata03:
#    driver: local
#  kibanadata:
#    driver: local

a2、同目录下编写 .env文件

# Password for the 'elastP@sswordic' user (at least 6 characters)
ELASTIC_PASSWORD=P@ssword

# Password for the 'kibana_system' user (at least 6 characters)
KIBANA_PASSWORD=P@ssword

# Version of Elastic products
STACK_VERSION=8.6.2

# Set the cluster name
CLUSTER_NAME=es_cluster

# Set to 'basic' or 'trial' to automatically start the 30-day trial
LICENSE=basic
#LICENSE=trial

# Port to expose Elasticsearch HTTP API to the host
ES_PORT=9200
#ES_PORT=127.0.0.1:9200

# Port to expose Kibana to the host
KIBANA_PORT=5601
#KIBANA_PORT=80

# Increase or decrease based on the available host memory (in bytes)
MEM_LIMIT=1073741824

# Project namespace (defaults to the current folder name if not set)
#COMPOSE_PROJECT_NAME=myproject

a3、后台启动

docker compose up -d   ## 后台运行

a4、查看集群健康状态

https://192.168.174.130:9200/_cluster/health?pretty

在这里插入图片描述

b、docker-compose.yaml shell脚本注解

检查环境变量ELASTIC_PASSWORD和KIBANA_PASSWORD是否设置，如果没有设置，则输出提示信息并退出脚本。
检查是否已经生成了CA和证书。如果没有生成，则使用elasticsearch-certutil工具生成，将生成的CA和证书文件保存在config/certs目录下。
设置文件权限为root:root和750/640，以保护文件安全。
等待Elasticsearch可用，并检查是否需要身份验证。使用curl命令检查Elasticsearch是否可用，如果返回的信息中包含"missing authentication credentials"字样，则说明不需要身份验证。此处等待30秒，直到Elasticsearch可用。
设置Kibana_system用户的密码。使用curl命令修改Kibana_system用户的密码，直到返回的信息中包含空json {
    
    } 为止。
输出"All done!"表示脚本执行完成。

c、health check注解

test：指定健康检查的命令或脚本，使用的格式为 CMD-SHELL，表示在容器内执行 Shell 命令。
interval：健康检查的间隔时间，默认为 30s。
timeout：每次健康检查的超时时间，默认为 30s。
retries：健康检查失败后的重试次数，默认为 3 次。

d、 depends_on注解

用于指定服务之间的依赖关系，其中depends_on用于指定本服务依赖的其他服务，condition用于指定依赖关系的条件。

在这段配置中，depends_on指定了本服务依赖的服务名为setup，condition指定了当setup服务的健康检查状态为service_healthy时，才会启动本服务。这意味着，本服务启动前会等待setup服务健康检查通过。

e、mem_limit资源限制

mem_limit: 容器最大内存限制。这里使用变量 ${MEM_LIMIT} 来设置内存限制，该变量的值应该在部署前设置。
ulimits: 包含容器的ulimit参数配置，用于控制容器对系统资源的使用。这里的 memlock 参数设置内存锁定限制，将内存锁定限制设置为无限制。
deploy: 用于定义Docker服务的部署策略和资源配置，包括：
resources: 用于指定该服务可以使用的CPU和内存资源的上限和下限。在这个例子中，设置CPU使用限制为0.5，内存限制为1G。同时，设置CPU使用保证为0.25，内存保证为512M。
注意这里的 limits 和 reservations 是两个不同的限制级别，限制级别的不同会影响Docker的调度策略。

3. 部署Logstash容器

我们将使用Logstash来收集、处理和转换数据，并将其发送到Elasticsearch。在这里，我们使用映射配置文件的方式来部署Logstash。

准备Logstash配置文件

cat logstash.yml

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "https://es01:9200" ]
xpack.monitoring.elasticsearch.username: "elastic"
xpack.monitoring.elasticsearch.password: "P@ssword"
xpack.monitoring.elasticsearch.ssl.certificate_authority: "/usr/share/logstash/config/ca.crt"

cat logstash.conf

input {
  beats {
    port => 5044
  }
}

filter {
  grok {
    match => { "message" => "%{COMBINEDAPACHELOG}" }
  }
}

output {
  elasticsearch {
    hosts => "https://es01:9200"
    user => "elastic"
    password => "P@ssword"
    ssl => true
    cacert => "/user/share/logstash/config/ca.crt"
    index => "httpd_logstash-%{+YYYY.MM.dd}"
  }
}

这个配置文件定义了Logstash如何处理收集到的日志数据。其中：

input定义了使用Beats协议监听5044端口来接收数据。
filter定义了如何将原始的日志数据进行过滤和转换，这里使用Grok插件匹配Apache日志格式。
output定义了将处理后的日志数据输出到Elasticsearch的方式。

运行logstash

docker run  -d \
-v /u01/elk/logstash/config/logstash.conf:/usr/share/logstash/config/logstash.conf \
-v /u01/elk/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml \
-v /u01/elk/logstash/config/ca.crt:/usr/share/logstash/config/ca.crt --name logstash --network elasticsearch_default  logstash:8.6.2 ./bin/logstash -f /usr/share/logstash/config/your_config_path

4. 部署Filebeat容器

4.1 准备Filebeat配置文件

我们将在主机上创建一个名为filebeat.yml的配置文件，并将其映射到容器内的路径。

filebeat.inputs:
- type: log
  paths:
    - /u01/elk/filebeat/logs/*.log

output.logstash:
  hosts: ["logstash:5044"]

这个配置文件定义了Filebeat如何收集和处理日志数据。其中：

filebeat.inputs定义了Filebeat要监控的日志文件路径。
output.logstash定义了将日志数据发送到Logstash的方式。

4.2 启动Filebeat容器

docker run -d --name filebeat \
    --network elasticsearch_default \
    -v /u01/elk/to/filebeat.yml:/usr/share/filebeat/filebeat.yml \
    -v /var/lib/docker/containers:/var/lib/docker/containers:ro \
    filebeat:8.6.2

在这个命令中，我们使用docker run命令创建并启动一个名为filebeat的容器。–network选项将容器连接到之前创建的elasticsearch_default网络。-v选项将主机上的Filebeat配置文件映射到容器内的路径。此外，我们还将Docker容器的日志路径映射到Filebeat容器内的路径，以便可以收集和处理Docker容器的日志数据。

5. 启动Kibana容器

最后，我们将在Docker容器中启动Kibana来可视化和分析Elasticsearch中的日志数据。

docker run -d --name kibana \
    -p 5601:5601 \
    --network elasticsearch_default \
    kibana:8.6.2

在这个命令中，我们使用docker run命令创建并启动一个名为kibana的容器。-p选项将Kibana容器的端口映射到主机的端口，–network选项将容器连接到之前创建的elasticsearch_default网络。

三、配置文件

logstash:

配置查询文档：https://www.elastic.co/guide/en/logstash/current/index.html
examples—> options

conf match=模式

[root@hh patterns]# pwd
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-patterns-core-4.3.4/patterns

logstash.conf

input{
    
    
  beats {
    
    
    port => 5044
  }
  file {
    
    
    path => [ "/usr/local/nginx/logs/access.log","/usr/local/nginx/logs/error.log" ]
    sincedb_path => "/var/log/logstash/since.db"
    start_position => "beginning"
    type => "nginxlog"
  }
  tcp {
    
    
    mode => "server"
    host => "0.0.0.0"
    port => 8888
    type => "tcplog"
  }
  udp {
    
    
    port => 8888
    type => "udplog"
      }
}
filter{
    
    
  grok {
    
    
       # match => {
    
     "message" => "%{
    
    IP:client} %{
    
    WORD:method} %{
    
    URIPATHPARAM:request} %{
    
    NUMBER:bytes} %{
    
    NUMBER:duration}" }
       match => {
    
     "message" => "(?<ip>[0-9\.]+).*\[(?<time>.+)\] \"(?<method>[A-Z]+) (?<url>\S+) (?<ver>[^\"]+)\" (?<rc>\d+) (?<size>\d+) \".*\" \"(?<agent>.*)\""}
      }
}
output{
    
    
  #stdout{
    
     codec => "json"}
  stdout{
    
     codec => "rubydebug"}
}

elasticsearch

elasticsearch.yml

[root@hh elasticsearch]# grep -Pv "^(#|$)" elasticsearch.yml 
node.name: es1              ##hosts文件中主机名
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 192.168.174.130      ###0.0.0.0监听地址
discovery.seed_hosts: ["es1"]    ###集群多个节点
cluster.initial_master_nodes: ["es1"]  
http.cors.enabled: true
http.cors.allow-origin: "*"

kibana

#server.port: 5601
server.host: '0.0.0.0'
#server.basePath: ""
#server.maxPayloadBytes: 1048576
#server.name: "your-hostname"
elasticsearch.url: 'http://192.168.174.130:9201'
#elasticsearch.preserveHost: true
#kibana.index: ".kibana"
#kibana.defaultAppId: "discover"
#elasticsearch.username: "user"
#elasticsearch.password: "pass"
#server.ssl.enabled: false
#server.ssl.certificate: /path/to/your/server.crt
#server.ssl.key: /path/to/your/server.key
#elasticsearch.ssl.certificate: /path/to/your/client.crt
#elasticsearch.ssl.key: /path/to/your/client.key
#elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ]
#elasticsearch.ssl.verificationMode: full
#elasticsearch.pingTimeout: 1500
#elasticsearch.requestTimeout: 30000
#elasticsearch.requestHeadersWhitelist: [ authorization ]
#elasticsearch.customHeaders: {
    
    }
#elasticsearch.shardTimeout: 0
#elasticsearch.startupTimeout: 5000
#pid.file: /var/run/kibana.pid
#logging.dest: stdout
#logging.silent: false
#logging.quiet: false
#logging.verbose: false
#ops.interval: 5000
i18n.defaultLocale: "zh_CN"

猜你喜欢

转载自blog.csdn.net/qq_44637753/article/details/128865756
今日推荐
《美国对全球网络空间安全与发展的威胁和破坏》报告发布
火速冲上 GitHub 热榜 —— 开源编程语言、框架哪有这么可爱？
北京人形机器人创新中心发布全球首个纯电驱拟人奔跑的全尺寸人形机器人“天工”
LFOSSA 源来如此公开课 | 掌握云原生未来：CNCF 认证全面攻略与备考秘籍
周排行
让自己的头脑极度开放
CentOS 6.5(x64) 和Redhat6.5操作系误删libc
高可用注册中心
【日记】12.28/【题解】AtCoder AGC041
XML（5）_XML 约束_DTD
Java集合Map（四）
树梅派安装桌面环境教程
pipenv 的 使用和安装
小程序白屏问题和内存研究
C语言简单选择排序
每日归档
更多
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)
2024-04-29(40)
2024-04-28(0)
2024-04-27(56)
2024-04-26(39)
2024-04-25(22)
2024-04-24(36)
2024-04-23(26)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022