SpringBoot2.0.2.RELEASE集成ElasticSearch6.6.1(RestHighLevelClient)

最近项目有用到Elasticsearch（以下简称ES），用了官方提供的RestHighLevelClient，在这里记录一下。

首先ES是有专门的同事在维护，我的应用层只是调用他们的服务，来进行日志检索。

第一步：引入Elasticsearch依赖

<properties>
    <elasticSearch.version>6.6.1</elasticSearch.version>
</properties>

<dependency>
    <groupId>org.elasticsearch</groupId>
    <artifactId>elasticsearch</artifactId>
    <version>${elasticSearch.version}</version>
</dependency>
<dependency>
    <groupId>org.elasticsearch.client</groupId>
    <artifactId>elasticsearch-rest-high-level-client</artifactId>
    <version>${elasticSearch.version}</version>
</dependency>
<dependency>
    <groupId>org.elasticsearch.plugin</groupId>
    <artifactId>transport-netty4-client</artifactId>
    <version>${elasticSearch.version}</version>
</dependency>

注意版本号需要严格对应哦。

第二步：创建配置类ESConfig

import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig.Builder;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestClientBuilder.HttpClientConfigCallback;
import org.elasticsearch.client.RestClientBuilder.RequestConfigCallback;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.ArrayList;

@Configuration
public class ESConfig {

    private static String hosts = "10.xx.xxx.xxx"; // 集群地址，多个用,隔开
    private static int port = xxxx; // 使用的端口号
    private static String schema = "http"; // 使用的协议
    private static ArrayList<HttpHost> hostList = null;

    private static int connectTimeOut = 1000; // 连接超时时间
    private static int socketTimeOut = 30000; // 连接超时时间
    private static int connectionRequestTimeOut = 500; // 获取连接的超时时间

    private static int maxConnectNum = 100; // 最大连接数
    private static int maxConnectPerRoute = 100; // 最大路由连接数

    static {
        hostList = new ArrayList<>();
        String[] hostStrs = hosts.split(",");
        for (String host : hostStrs) {
            hostList.add(new HttpHost(host, port, schema));
        }
    }

    @Bean
    public RestHighLevelClient client(){
        RestClientBuilder builder = RestClient.builder(hostList.toArray(new HttpHost[0]));
        // 异步httpclient连接延时配置
        builder.setRequestConfigCallback(new RequestConfigCallback() {
            @Override
            public Builder customizeRequestConfig(Builder requestConfigBuilder) {
                requestConfigBuilder.setConnectTimeout(connectTimeOut);
                requestConfigBuilder.setSocketTimeout(socketTimeOut);
                requestConfigBuilder.setConnectionRequestTimeout(connectionRequestTimeOut);
                return requestConfigBuilder;
            }
        });
        // 异步httpclient连接数配置
        builder.setHttpClientConfigCallback(new HttpClientConfigCallback() {
            @Override
            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                httpClientBuilder.setMaxConnTotal(maxConnectNum);
                httpClientBuilder.setMaxConnPerRoute(maxConnectPerRoute);
                return httpClientBuilder;
            }
        });
        RestHighLevelClient client = new RestHighLevelClient(builder);
        return client;
    }
}

第三步：创建功能类

查询：

首先我们得看下ES服务那边提供的数据的Json结构：

"hits" : {
    "total" : 12,
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "filebeat-6.6.1-2019.03.21",
        "_type" : "doc",
        "_id" : "d6Jan2kBXvHfQ1BtNqYz",
        "_score" : 1.0,
        "_source" : {
          "offset" : 3348,
          "log" : {
            "file" : {
              "path" : "/xxx/xxx/log/redis.log"
            }
          },
          "prospector" : {
            "type" : "log"
          },
          "read_timestamp" : "2019-03-21T08:24:37.873Z",
          "source" : "/xxx/xxx/log/redis.log",
          "fileset" : {
            "module" : "redis",
            "name" : "log"
          },
          "redis" : {
            "log" : {
              "role" : "master",
              "level" : "warning",
              "pid" : "11027",
              "message" : "User requested shutdown..."
            }
          },
          "input" : {
            "type" : "log"
          },
          "@timestamp" : "2019-03-21T16:24:33.477Z",
          "beat" : {
            "hostname" : "xxx",
            "name" : "xxx",
            "version" : "6.6.1"
          },
          "host" : {
            "name" : "xxx"
          },
          "event" : {
            "dataset" : "redis.log"
          },
          "fields" : {
            "port" : 6410,
            "entity_id" : 319
          }
        }
      }
    ]
  }

假如是这种比较复杂的Json结构，我们要查询entity_id是319，时间戳read_timestamp在某一个区间里这几个字段redis.log.role，log.file.path，fields.port，fields.entity_id，redis.log.message的值。

import com.alibaba.fastjson.JSON;
import com.foresealife.dbaas.entity.common.ActionResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.IOException;

@Service
public class ElasticSearchLogService {
    private static final Logger LOG = LoggerFactory.getLogger(ElasticSearchLogService.class);

    @Autowired
    private RestHighLevelClient client;

    public ActionResponse search() throws IOException {
        BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery();
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("fields.entity_id", "319");//这里可以根据字段进行搜索，must表示符合条件的，相反的mustnot表示不符合条件的
        RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("read_timestamp"); //新建range条件
        rangeQueryBuilder.gte("2019-03-21T08:24:37.873Z"); //开始时间
        rangeQueryBuilder.lte("2019-03-21T08:24:37.873Z"); //结束时间
        boolBuilder.must(matchQueryBuilder);
        boolBuilder.must(rangeQueryBuilder);
        sourceBuilder.query(boolBuilder);
        sourceBuilder.from(0);
        sourceBuilder.size(100); // 获取记录数，默认10
        sourceBuilder.fetchSource(new String[] {"redis.log.role","log.file.path","fields.port","fields.entity_id","redis.log.message"}, new String[] {}); //第一个是获取字段，第二个是过滤的字段，默认获取全部
        SearchRequest searchRequest = new SearchRequest("filebeat-6.6.1-2019.03.21"); //索引
        searchRequest.types("doc"); //类型
        searchRequest.source(sourceBuilder);
        SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
        LOG.info("search: {}",JSON.toJSONString(response));
        SearchHits hits = response.getHits();
        SearchHit[] searchHits = hits.getHits();
        for (SearchHit hit : searchHits) {
            LOG.info("search -> {}",hit.getSourceAsString());
        }
        return ActionResponse.actionSuccess("","");
    }
}

以上是查询的功能，后续加入更多ES功能后，再补充。