windows部署filebeat+logstash+ES

ES

1. elasticsearch-6.5.3\plugins，放入analysis-ik中文分词插件
2. 运行bin/elasticsearch.bat，端口9200
3. 部署es head插件，端口9100

logstash

4. logstash-6.5.3\config，新建conf文件myconfig.conf

input {
    
    
	beats{
    
    
			port => "5044"
        }
}
output {
    
    
        stdout {
    
    
                codec => rubydebug
        }
		 elasticsearch {
    
    
    hosts => "localhost:9200" 
    index => "log-%{+YYYY.MM.dd}"


  }
}

PS：从filebeat输入，输入到终端显示和ES中

2. bin目录下执行，logstash.bat -f …/config/myconfig.conf，端口9600

filebeat

1. 修改filebeat.yml

filebeat.inputs:
 1. type: log
  enabled: true
  paths:
    - E:\workspace2\spring-security-top\spring-security-back\log\*.log
output.logstash:
  hosts: ["localhost:5044"]
# 其他配置省略  

PS：path为输入的日志文件；默认输出到ES，这里修改为logstash

2. 执行filebeat -e -c filebeat.yml -d “publish”
3. 如果日志有更新，会在终端显示

2019-12-06T13:20:21.948+0800    DEBUG   [publish]       pipeline/processor.go:308       Publish event: {
    
    
  "@timestamp": "2019-12-06T05:20:21.948Z",
  "@metadata": {
    
    
    "beat": "filebeat",
    "type": "doc",
    "version": "6.5.3"
  },
  "host": {
    
    
    "name": "DESKTOP-2OR9HPK",
    "id": "0eac6134-4ede-4fb4-9208-d21d113adcd2",
    "architecture": "x86_64",
    "os": {
    
    
      "platform": "windows",
      "version": "10.0",
      "family": "windows",
      "build": "17763.864"
    }
  },
  "message": "test6",
  "source": "E:\\workspace2\\spring-security-top\\spring-security-back\\log\\error.2019-12-06.log",
  "offset": 54,
  "prospector": {
    
    
    "type": "log"
  },
  "input": {
    
    
    "type": "log"
  },
  "beat": {
    
    
    "hostname": "DESKTOP-2OR9HPK",
    "version": "6.5.3",
    "name": "DESKTOP-2OR9HPK"
  }
}

4. 查看es head，结果已经输入到ES中

一般流程架构

filebeat采集数据=》redis或kafka等做消息队列=》logstash消息转换=》放入ES=》kibana可视化
以redis为例
filebeat.yml

output.redis:
  hosts: ["localhost"]    #输出到redis的机器
  port: 6379    #redis端口号
  db: 2    #redis数据库的一个整数索引标识
  password: 123456 #密码
  timeout: 5    #连接超时时间
  key: "default_list"    #以default_list的keys传输到redis

logstash配置

input {
    redis {
        data_type => "list"
        key => "ldlog"
        host => "localhost"
        port => "6379"
        db => "2"
        password => "123456"
	}
}
output {
        stdout {
                codec => rubydebug
        }
	elasticsearch {
		hosts => "localhost:9200" 
		index => "log-%{+YYYY.MM.dd}"
  }
}