如何使用solr来构建一个大数据平台？

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/qq_36520235/article/details/82113586

如何实现使用solr构建大数据平台的思路（其实跟普通的项目差距不是特别大）

1. 首先需要构建一个从web后台传给大数据平台的一个Json对象
2. 你需要在大数据平台的项目中也是先需要定义一个controller层来进行处理

（一） 需要先对solr的查询条件进行构造的函数工具类

package com.ia.bigdata.car.component.solr;

import java.util.List;

import org.apache.commons.lang3.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;

/**
 * 查询构建器
 * @author IAP-liuc
 *
 */
public class SolrConstructor {

    private SolrQuery solrQuery;

    public final static String EQUAL = "equal";

    public final static String GREATER = "greater";

    public final static String LESS = "less";

    public final static String INCLUDE = "include";

    public final static String NOTINCLUDE = "notclude";

    public final static String RANGE = "range";

    public final static String LIKE = "like";

    public final static String STARTWITH = "startwith";

    public final static int LIKE_LEVEL_1 = 1;

    public final static int LIKE_LEVEL_2 = 2;

    public final static int LIKE_LEVEL_3 = 3;

    public final static int LIKE_LEVEL_4 = 4;

    public final static int LIKE_LEVEL_5 = 5;

    /**
     * 首次构建
     */
    public SolrConstructor() {
        solrQuery = new SolrQuery();
        solrQuery.setQuery("*:*");
    }

    /**
     * 基于已经存在的条件构建
     * 
     * @param solrQuery
     */
    public SolrConstructor(SolrQuery solrQuery) {
        this.solrQuery = solrQuery;
    }

    /**
     * 返回查询条件
     * @return
     */
    public SolrQuery sq() {
        return this.solrQuery;
    }


    /**
     * 单参数比较构造
     * 
     * @param oper
     * @param col
     * @param param
     * @return
     */
    public SolrConstructor construct(String oper, String col, String param) {
        StringBuilder queryBuilder = new StringBuilder("");
        if (SolrConstructor.EQUAL.equals(oper)) {
            queryBuilder.append(col + ":" + param);
        } else if (SolrConstructor.GREATER.equals(oper)) {
            queryBuilder.append(col + ":" + "[" + param + " TO *]");
        } else if (SolrConstructor.LESS.equals(oper)) {
            queryBuilder.append(col + ":" + "[* TO " + param + "]");
        }else if (SolrConstructor.STARTWITH.equals(oper)) {
            queryBuilder.append(col + ":" + param + "~");
        }
        solrQuery.addFilterQuery(queryBuilder.toString());
        return this;
    }

    /**
     * 相似参数构建
     * 
     * @param oper
     * @param col
     * @param param
     * @return
     */
    public SolrConstructor construct(String oper, int level, String col, String param) {
        StringBuilder queryBuilder = new StringBuilder("");
        if (SolrConstructor.LIKE.equals(oper)) {
            queryBuilder.append(col + ":" + param + "~" + level);
        }
        solrQuery.addFilterQuery(queryBuilder.toString());
        return this;
    }

    /**
     * 复合参数构建
     * 
     * @param oper
     * @param col
     * @param param
     * @return
     */
    public SolrConstructor construct(String oper, String col, List<String> params) {
        StringBuilder queryBuilder = new StringBuilder("");
        if (SolrConstructor.INCLUDE.equals(oper)) {
            if (params != null && params.size() > 0) {
                for (String str : params) {
                    if (queryBuilder.toString().equals("")) {
                        queryBuilder.append(col + ":" + str);
                    } else {
                        queryBuilder.append(" OR " + col + ":" + str);
                    }
                }
            }
        } else if (SolrConstructor.NOTINCLUDE.equals(oper)) {
            // TODO : 不包含功能暂未实现
        }
        solrQuery.addFilterQuery(queryBuilder.toString());
        return this;
    }

    /**
     * 复合参数构建
     * 
     * @param oper
     * @param col
     * @param param
     * @return
     */
    public SolrConstructor construct(String oper, String col, String[] params) {
        StringBuilder queryBuilder = new StringBuilder("");
        if (SolrConstructor.INCLUDE.equals(oper)) {
            if (params != null && params.length > 0) {
                for (String str : params) {
                    if (queryBuilder.toString().equals("")) {
                        queryBuilder.append(col + ":" + str);
                    } else {
                        queryBuilder.append(" OR " + col + ":" + str);
                    }
                }
            }
        } else if (SolrConstructor.NOTINCLUDE.equals(oper)) {
            // TODO : 不包含功能暂未实现
        }
        solrQuery.addFilterQuery(queryBuilder.toString());
        return this;
    }

    /**
     * 范围参数构建
     * @param oper
     * @param col
     * @param rangeStart
     * @param rangeEnd
     * @return
     */
    public SolrConstructor construct(String oper, String col, String rangeStart, String rangeEnd) {
        StringBuilder queryBuilder = new StringBuilder("");
        if (SolrConstructor.RANGE.equals(oper)) {
            if (StringUtils.isNoneEmpty(rangeStart) && StringUtils.isNoneEmpty(rangeEnd)) {
                queryBuilder.append(col + ":[" + rangeStart + " TO " + rangeEnd + "]");
                solrQuery.addFilterQuery(queryBuilder.toString());
            }
        }
        return this;
    }

    /**
     * 构建查询column
     * @param field
     * @return
     */
    public SolrConstructor setField(String field) {
        solrQuery.setFields(field);
        return this;
    } 
}

（二） 大数据

package com.ia.bigdata.car.controller.techtactics;

import com.ia.bigdata.car.component.jrv.JRV;
import com.ia.bigdata.car.component.log.Log4jKit;
import com.ia.bigdata.car.module.constvalue.SystemConst;
import com.ia.bigdata.car.module.job.model.JobEntity;
import com.ia.bigdata.car.module.job.model.JobParamterContinuousViolationEntity;
import com.ia.bigdata.car.module.techtactics.service.TechtacticsJobManagerService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

/**
 * Created by IAP-wanghan on 2018/5/22.
 */

@RestController
@RequestMapping("techtactics/continuousviolation")
public class TechTacticsContinuousViolationController {
    @Autowired
    private TechtacticsJobManagerService techtacticsJobManagerService;

    @RequestMapping("start")
    public JRV<Object> doGetContinuousViolation(@RequestBody JobParamterContinuousViolationEntity entity) {
        JRV<Object> jrv = new JRV<Object>();
        JobEntity job = null;
        try {
            job = techtacticsJobManagerService.startJob(SystemConst.TECHTACTICS_TYPE_CONTINUOUSVIOLATION, entity);
            jrv.push(job.getId());
            return jrv;
        } catch (Exception e) {
            Log4jKit.error("异常："+ e.toString());
            jrv.fault(e);
        }
        return jrv;
    }
}

（三） 最后返回给前台项目的实体

package com.ia.bigdata.car.component.jrv;

import java.io.Serializable;

public class JRV<T> implements Serializable{

    /**
     * 
     */
    private static final long serialVersionUID = 8667203039777652887L;

    private String result; 

    private T context;

    private String errorMsg;

    public void push(T t) {
        if (t instanceof Result) {
            result = t.toString();
            context = t;
        } else {
            result = Result.OK.toString();
            context = t;
        }
    }

    public void fault(Exception e) {
        errorMsg = e.getMessage();
    }

    public String getResult() {
        return result;
    }

    public void setResult(String result) {
        this.result = result;
    }

    public T getContext() {
        return context;
    }

    public void setContext(T context) {
        this.context = context;
    }

    public String getErrorMsg() {
        return errorMsg;
    }

    public void setErrorMsg(String errorMsg) {
        this.errorMsg = errorMsg;
    }
}

（四）大数据平台的service层

package com.ia.bigdata.car.module.techtactics.service;

import java.util.List;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import com.ia.bigdata.car.module.job.model.JobEntity;

@Service
public class TechtacticsJobManagerService {

    @Autowired
    private List<ITechtacticsJobService> techtacticsJobServices;

    private ITechtacticsJobService getService(int type) {
        for(ITechtacticsJobService service : techtacticsJobServices) {
            if(type == service.getJobType()) {
                return service;
            }
        }
        return null;
    }

    public <T> T getJobParamterData(Object param) {
        //TODO: change patam to data
        return null;
    }

    public <T> JobEntity startJob(int type, T t) throws Exception {
        ITechtacticsJobService service = getService(type);
        if(null != service) {
            return service.exec(type, t);
        } else {
            throw new Exception();
        }
    }
}

（五） 任务参数实体、dao接口、dao实现类

package com.ia.bigdata.car.module.job.model;

import java.io.Serializable;

/**
 * 任务参数实体
 * @author IAP-liuc
 *
 */
public class JobParamterContinuousViolationEntity implements Serializable{

    /**
     * 
     */
    private static final long serialVersionUID = 1L;

    private Long id;

    private Long jobid;

    private String starttime;

    private String endtime;

    private String devices;

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public Long getJobid() {
        return jobid;
    }

    public void setJobid(Long jobid) {
        this.jobid = jobid;
    }

    public String getStarttime() {
        return starttime;
    }

    public void setStarttime(String starttime) {
        this.starttime = starttime;
    }

    public String getEndtime() {
        return endtime;
    }

    public void setEndtime(String endtime) {
        this.endtime = endtime;
    }

    public String getDevices() {
        return devices;
    }

    public void setDevices(String devices) {
        this.devices = devices;
    }


}

package com.ia.bigdata.car.module.job.dao;

import com.ia.bigdata.car.module.job.model.JobParamterContinuousViolationEntity;
import com.ia.bigdata.car.module.techtactics.model.ContinuousViolationResultEntity;
import com.ia.bigdata.car.module.techtactics.model.abnormalstay.AbnormalStayModelEntity;
import org.apache.ibatis.annotations.Param;

import java.util.List;

public interface JobContinuousViolationParamterDao {

    //连续违章
    /**
     *
     * @param jobParam
     * @return
     */
    int insert(JobParamterContinuousViolationEntity jobParam);
    /**
     *
     * @param starttime formate yyyymmdd
     * @param endtime formate yyyymmdd
     * @return
     */

    List<JobParamterContinuousViolationEntity> selectContinuousViolationParamtersByStartAndEnd(@Param("starttime") String starttime, @Param("endtime") String endtime);

    /**
     * 批量生成结果
     * @param results
     */
    void insertResultBatch(List<ContinuousViolationResultEntity> results);

    List<ContinuousViolationResultEntity> selectViolationInfo(@Param("starttime") String starttime, @Param("endtime") String endtime, @Param("deviceList") List<String> deviceList);
}

package com.ia.bigdata.car.module.job.service.impl;

import com.ia.bigdata.car.module.constvalue.SystemConst;
import com.ia.bigdata.car.module.job.dao.JobContinuousViolationParamterDao;
import com.ia.bigdata.car.module.job.dao.JobDao;
import com.ia.bigdata.car.module.job.model.JobEntity;
import com.ia.bigdata.car.module.job.model.JobParamterContinuousViolationEntity;
import com.ia.bigdata.car.module.job.service.JobGenerate;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

@Component
public class JobGenerateContinuousViolationImpl extends JobGenerate<JobParamterContinuousViolationEntity> {

    @Autowired
    private JobContinuousViolationParamterDao jobContinuousViolationParamterDao;
    @Autowired
    private JobDao jobDao;

    @Override
    public JobEntity createJob(JobParamterContinuousViolationEntity paramContinuousViolation) {
        JobEntity job = isExist(paramContinuousViolation);
        if(null != job) {
            job.setExist(true);
            return job;
        } else {
            JobEntity newjob = new JobEntity();
            newjob.setTtype(SystemConst.TECHTACTICS_TYPE_CONTINUOUSVIOLATION);
            newjob.setTstatus(SystemConst.TECHTACTICS_JOB_RUN_STATUS_CREATE);
            newjob.setCreatedate(new Date());
            jobDao.insert(newjob);

            paramContinuousViolation.setJobid(newjob.getId());
            jobContinuousViolationParamterDao.insert(paramContinuousViolation);
            return newjob;
        }
    }

    @Override
    protected JobEntity isExist(JobParamterContinuousViolationEntity paramContinuousViolation) {
        String starttime = paramContinuousViolation.getStarttime();
        String endtime = paramContinuousViolation.getEndtime();
        String deviceids = paramContinuousViolation.getDevices();
        Map<String , String> devmap = new HashMap<String , String>();

        if(StringUtils.isNoneEmpty(deviceids)) {
            String[] devs = deviceids.split(",");
            for(String dev : devs) {
                devmap.put(dev, dev);
            }
        }

        List<JobParamterContinuousViolationEntity> jobparams = jobContinuousViolationParamterDao.selectContinuousViolationParamtersByStartAndEnd(starttime, endtime);
        if(null != jobparams && jobparams.size() > 0) {
            for(JobParamterContinuousViolationEntity entity : jobparams) {
                String devs = entity.getDevices();
                if(StringUtils.isNoneEmpty(devs)) {
                    String[] existdevs = devs.split(",");
                    if(existdevs.length == devmap.keySet().size()) {
                        boolean exist = true;
                        for(String existdev : existdevs) {
                            if(!devmap.containsKey(existdev)) {
                                exist = false;
                                break;
                            }
                        }

                        if(exist == true) {
                            Long jobId = entity.getJobid();
                            JobEntity job = jobDao.selectById(jobId);
                            if(SystemConst.TECHTACTICS_JOB_RUN_STATUS_SCUESS == job.getTstatus()) {
                                return job;
                            }
                        }
                    }
                }
            }
        }
        return null;
    }


}

返回实体类

package com.ia.bigdata.car.module.techtactics.model;

import java.io.Serializable;

public class ContinuousViolationResultEntity implements Serializable {

    /**
     *
     */
    private static final long serialVersionUID = 1L;

    private Long id;

    private Long jobid;

    private String platenumber;

    private String platecolor;

    private String vehicletype;

    private String datetime;

    private String deviceID;

    private String address;

    private String direction;

    private String violationtype;

    private Long violationcount;

    private String picpath;


    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public Long getJobid() {
        return jobid;
    }

    public void setJobid(Long jobid) {
        this.jobid = jobid;
    }

    public String getPlatenumber() {
        return platenumber;
    }

    public void setPlatenumber(String platenumber) {
        this.platenumber = platenumber;
    }

    public String getPlatecolor() {
        return platecolor;
    }

    public void setPlatecolor(String platecolor) {
        this.platecolor = platecolor;
    }

    public String getVehicletype() {
        return vehicletype;
    }

    public void setVehicletype(String vehicletype) {
        this.vehicletype = vehicletype;
    }

    public String getDatetime() {
        return datetime;
    }

    public void setDatetime(String datetime) {
        this.datetime = datetime;
    }

    public String getDeviceID() {
        return deviceID;
    }

    public void setDeviceID(String deviceID) {
        this.deviceID = deviceID;
    }

    public String getAddress() {
        return address;
    }

    public void setAddress(String address) {
        this.address = address;
    }

    public String getDirection() {
        return direction;
    }

    public void setDirection(String direction) {
        this.direction = direction;
    }

    public String getViolationtype() {
        return violationtype;
    }

    public void setViolationtype(String violationtype) {
        this.violationtype = violationtype;
    }

    public Long getViolationcount() {
        return violationcount;
    }

    public void setViolationcount(Long violationcount) {
        this.violationcount = violationcount;
    }

    public String getPicpath() {
        return picpath;
    }

    public void setPicpath(String picpath) {
        this.picpath = picpath;
    }
}

连续违章

package com.ia.bigdata.car.module.techtactics.service.impl;

import com.ia.bigdata.car.component.exception.BusinessException;
import com.ia.bigdata.car.component.log.Log4jKit;
import com.ia.bigdata.car.module.constvalue.SystemConst;
import com.ia.bigdata.car.module.job.dao.JobContinuousViolationParamterDao;
import com.ia.bigdata.car.module.job.dao.JobDao;
import com.ia.bigdata.car.module.job.dao.JobFreqNightCarParamterDao;
import com.ia.bigdata.car.module.job.model.JobEntity;
import com.ia.bigdata.car.module.job.model.JobParamterContinuousViolationEntity;
import com.ia.bigdata.car.module.job.model.JobParamterFreqNightCarEntity;
import com.ia.bigdata.car.module.job.service.impl.JobGenerateContinuousViolationImpl;
import com.ia.bigdata.car.module.job.service.impl.JobGenerateFreqNightCarImpl;
import com.ia.bigdata.car.module.techtactics.dao.freqnightcar.TTFreqNightCarDao;
import com.ia.bigdata.car.module.techtactics.model.ContinuousViolationResultEntity;
import com.ia.bigdata.car.module.techtactics.model.FreqNightCarResultEntity;
import com.ia.bigdata.car.module.techtactics.service.ITechtacticsJobService;
import org.apache.solr.client.solrj.SolrServerException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * 连续违章
 * @author IAP-wanghan
 *
 */
@Component
public class TechtacticsContinuousViolationJobService implements ITechtacticsJobService{

    @Autowired
    private JobGenerateContinuousViolationImpl jobGenerateContinuousViolationImpl;

    @Autowired
    private JobContinuousViolationParamterDao jobContinuousViolationParamterDao;

    @Autowired
    private JobDao jobDao;

    String starttime;

    String endtime;

    String[] devices;

    @Override
    public int getJobType() {
        return SystemConst.TECHTACTICS_TYPE_CONTINUOUSVIOLATION;
    }

    @Override
    public <T> JobEntity exec(int type, T t) throws BusinessException {
        JobParamterContinuousViolationEntity paramContinuousViolation = (JobParamterContinuousViolationEntity)t;
        this.starttime = paramContinuousViolation.getStarttime();
        this.endtime = paramContinuousViolation.getEndtime();
        this.devices = paramContinuousViolation.getDevices().split(",");

        JobEntity job;

        DateFormat df = new SimpleDateFormat("yyyyMMddHHmmss");
        Calendar calendarStart = Calendar.getInstance();
        Calendar calendarEnd = Calendar.getInstance();

        try {
            calendarStart.setTime(df.parse(starttime));
            calendarEnd.setTime(df.parse(endtime));
        } catch (ParseException e) {
            Log4jKit.error(e.toString(), e);
        }

        if(calendarStart.compareTo(calendarEnd) >= 0) {
            Log4jKit.error("查询终止时间需在起始时间之后");
            throw new BusinessException("206");
        }else{
            job = jobGenerateContinuousViolationImpl.createJob(paramContinuousViolation);
            if (SystemConst.TECHTACTICS_JOB_RUN_STATUS_SCUESS != job.getTstatus()){
                DoContinuousViolationJob(paramContinuousViolation);
            }
        }

        return job;

    }
    /**
     * @开始执行连续违章任务
     * @param paramContinuousViolation
     * @return
     */
    private int DoContinuousViolationJob(JobParamterContinuousViolationEntity paramContinuousViolation) throws BusinessException {
        jobDao.updateStatusById(paramContinuousViolation.getJobid(), SystemConst.TECHTACTICS_JOB_RUN_STATUS_START);

        List<String> deviceList = Arrays.asList(devices);
        List<ContinuousViolationResultEntity> violationInfo = jobContinuousViolationParamterDao.selectViolationInfo(starttime, endtime, deviceList);
        Log4jKit.info("符合条件的所有违章信息条数为：" + violationInfo.size());

        if(violationInfo.size() == 0){
            Log4jKit.error("未在指定时间区域范围内查询到违章信息");
            throw new BusinessException("207");
        }

        Map<String, List<ContinuousViolationResultEntity>> violationInfoStat = new HashMap<>();
        for(ContinuousViolationResultEntity perViolationInfo: violationInfo){
            String platenumberAndColor = perViolationInfo.getPlatenumber() + "-" + perViolationInfo.getPlatecolor();
            if(!violationInfoStat.keySet().contains(platenumberAndColor)){
                List<ContinuousViolationResultEntity> violationInfoList = new ArrayList<>();
                violationInfoList.add(perViolationInfo);
                violationInfoStat.put(platenumberAndColor, violationInfoList);
            }else {
                violationInfoStat.get(platenumberAndColor).add(perViolationInfo);
            }
        }
        Log4jKit.info("符合条件的车数为：" + violationInfoStat.size());

        List<ContinuousViolationResultEntity> continuousViolationResultEntities = new ArrayList<>();
        for(String car : violationInfoStat.keySet()){
            if(violationInfoStat.get(car).size() >= 3){
                for(ContinuousViolationResultEntity continuousViolationResultEntity : violationInfoStat.get(car)){
                    continuousViolationResultEntity.setJobid(paramContinuousViolation.getJobid());
                    continuousViolationResultEntity.setViolationcount((long)violationInfoStat.get(car).size());
                    continuousViolationResultEntities.add(continuousViolationResultEntity);
                }
            }
        }

        if(continuousViolationResultEntities.size() == 0){
            Log4jKit.info("未识别出连续违章车辆");
        }else{
            WriteResultToDB(continuousViolationResultEntities);
        }

        jobDao.updateStatusById(paramContinuousViolation.getJobid(), SystemConst.TECHTACTICS_JOB_RUN_STATUS_SCUESS);

        return 2;
    }


    /**
     * @分析结果写入Mysql关系型数据库的iabc_ttr_continuousviolation表
     * @param continuousViolationResultEntityList
     */
    private void WriteResultToDB(List<ContinuousViolationResultEntity> continuousViolationResultEntityList){

        if (continuousViolationResultEntityList.size() > 0 ) {
            jobContinuousViolationParamterDao.insertResultBatch(continuousViolationResultEntityList);
            Log4jKit.info("数据成功写入数据库");
        }

    }
}

构建solr参数去查询

package com.ia.bigdata.car.module.techtactics.dao.firstentry;

import com.ia.bigdata.car.component.solr.BaseSolrHBaseDAO;
import com.ia.bigdata.car.component.solr.SolrConstructor;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.params.FacetParams;
import org.springframework.stereotype.Repository;

import java.io.IOException;
import java.util.List;
import java.util.Set;

@Repository
public class TTFirstEntryDao extends BaseSolrHBaseDAO {

    public String getCaseFirstEntryCars(String start, String end, String[] devices, Set<String> result)
            throws SolrServerException, IOException {

        SolrConstructor constructor = new SolrConstructor();

        if (StringUtils.isNoneEmpty(start) && StringUtils.isNoneEmpty(end)){
            constructor = constructor.construct(SolrConstructor.RANGE, this.COLUMN_DATATIME, start, end);
        }
        if (null != devices && devices.length > 0){
            constructor = constructor.construct(SolrConstructor.INCLUDE, this.COLUMN_DEVICEID, devices);
        }
        SolrQuery solrQuery = constructor.sq();
        solrQuery.setFacet(true);
        solrQuery.setFacetMinCount(1);
        solrQuery.setFacetLimit(-1);
        solrQuery.setFacetSort(FacetParams.FACET_SORT_COUNT);
        solrQuery.addFacetField(this.COLUMN_ID);

        System.out.println("Start Query");
        QueryResponse response = client().query(solrQuery, SolrRequest.METHOD.POST);
        List<FacetField> facetFields = response.getFacetFields();
        for (FacetField ff : facetFields){
            List<FacetField.Count> count = ff.getValues();
            for (FacetField.Count c : count){
                result.add(c.getName());
            }
        }

        return "";
    }
}