ELK配置

环境：
Centos7.4

软件：
jdk-1.8.0_211；
elasticsearch-7.9.2；
search-guard-7
logstash-7.9.2；
kibana-7.9.2；
filebeat-7.7.1

场景：
公司自研java应用，日志文件共三个，分别为appname_info.log、appname_warn.log、appname_error.log。其中info和warn日志格式一样，error日志为java应用报错日志。

难点一：由于配置了search-guard-7，logstash连接elasticsearch时认证报错，最终配置如下：

output {
    elasticsearch {
        hosts => ["192.168.20.39:9200", "192.168.20.40:9200", "192.168.20.41:9200"]
        ssl => true
        ssl_certificate_verification => false
        cacert => "/export/server/logstash-7.9.2/config/root-ca.pem"
        index => "ngms-exchange-%{+YYYY.MM.dd}"
        user => "admin"
        password => "admin"
    }
}

难点二：logstash匹配两种格式的日志，配置如下：

filter {
    grok {
        match => {
            "message" => [
                '(?<recod-time>\d+-\d+-\d+\s+\d+:\d+:\d+\.\d+)(\|)(?<log-level>\w+)(\s+\|)(?<log-message>.*)',  #匹配单行日志
                '(?m)^%{TIMESTAMP_ISO8601:recod-time}\|%{LOGLEVEL:log-level}\|%{GREEDYDATA:log-message}'  #匹配多行日志
            ]
        }
        remove_field => ["message","agent","flags","ecs","os","path","@version"]
    }

    date {
        match => ["recod-time","yyyy-MM-dd HH:mm:ss.SSS"]
        target => "@timestamp"  #用日志的时间替换logstash原来的timestamp
    }
}

难点三：将不同应用的日志使用filebeat输出到logstash，在filebeat配置中添加tags，然后在logstash中进行判断：

#filebeat配置如下：

- type: log
  paths:
    - /export/logs/appname1/appname1*.log
  fields:
    source: appname1
  scan_frequency: 10s
    #filebeat多行匹配
  multiline.pattern: '^\d{4}-\d{2}-\d{2}'
  multiline.negate: true
  multiline.match: after
    #添加tags
  tags: ['appname1']

- type: log
  paths:
    - /export/logs/appname2/appname2*.log
  fields:
    source: appname2
  scan_frequency: 10s
    #filebeat多行匹配
  multiline.pattern: '^\d{4}-\d{2}-\d{2}'
  multiline.negate: true
  multiline.match: after
    #添加tags
  tags: ['appname2']

#logstash配置

output {
    if "ngms-exchange" in [tags] {
        elasticsearch {
            hosts => ["192.168.20.39:9200", "192.168.20.40:9200", "192.168.20.41:9200"]
            ssl => true
            ssl_certificate_verification => false
            cacert => "/export/server/logstash-7.9.2/config/root-ca.pem"
            index => "ngms-exchange-%{+YYYY.MM.dd}"
            user => "admin"
            password => "admin"
        }
    }
}