SpringBoot 2.X集成Hive-jdbc 3.1.1

最近公司有一个需求,需求的内容是根据用户页面选择的参数条件查询Hive,数量量大致是300万以内,要求3秒响应.使用的其它的技术就不要说了,先说说SpingBoot集成Hive-jdbc吧,网上虽然有完整的集成方案,但是根据方案来实现总是遇到各种各样的问题,一会日志包问题 一会jetty问题,各种烦心的异常.这次蹭着这个机会来说说我是怎么集成的.

先贴上我的pom.xml相关依赖:

<dependency>
    <groupId>org.projectlombok</groupId>
    <artifactId>lombok</artifactId>
</dependency>
<dependency>
    <groupId>org.apache.hive</groupId>
    <artifactId>hive-jdbc</artifactId>
    <version>3.1.1</version>
    <exclusions>
        <exclusion>
            <groupId>org.eclipse.jetty</groupId>
            <artifactId>*</artifactId>
        </exclusion>
        <exclusion>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
        </exclusion>
        <exclusion>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-slf4j-impl</artifactId>
        </exclusion>
        <exclusion>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-shims</artifactId>
        </exclusion>
        <exclusion>
            <groupId>javax.servlet</groupId>
            <artifactId>servlet-api</artifactId>
        </exclusion>
    </exclusions>
</dependency>
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>

连接数据库肯定需要DataSource配置,我这里用的是SpringBoot默认的HikariCP.

@Data
@Configuration
@ConfigurationProperties(prefix = "hive")
public class HiveHikariConfig {

    private String url;
    private String user;
    private String password;
    private String driverClassName;

    @Bean(name = "hiveHikariDataSource")
    @Qualifier("hiveHikariDataSource")
    public DataSource dataSource() {
        HikariDataSource datasource = new HikariDataSource();
        datasource.setJdbcUrl(url);
        datasource.setUsername(user);
        datasource.setPassword(password);
        datasource.setDriverClassName(driverClassName);
        return datasource;
    }

    @Bean(name = "hiveHikariTemplate")
    public JdbcTemplate hiveDruidTemplate(@Qualifier("hiveHikariDataSource") DataSource dataSource) {
        return new JdbcTemplate(dataSource);
    }
}

配置完成以后,我们就需要写工具类了

HiveRepository.java

@Service
public class HiveRepository{

    @Autowired
    private JdbcTemplate hiveJdbcTemplate;

    /**
     * <li>Description: TODO </li>
     */
    @PostConstruct
    public void createTable() {
        /*建表SQL语句*/
        StringBuffer sql = new StringBuffer("create table IF NOT EXISTS ");
        sql.append("bus_receiver ");
        sql.append("(id BIGINT comment '主键ID' " +
                ",name STRING  comment '姓名' " +
                ",address STRING comment '地址'" +
                ",en_name STRING comment '拼音名字'" +
                ",member_family INT comment '家庭成员'" +
                ",createDate DATE comment '创建时') ");
        sql.append(" ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'"); // 定义分隔符
        sql.append(" STORED AS TEXTFILE"); // 作为文本存储*/
        hiveJdbcTemplate.execute(sql.toString());
    }

    /**
     * <li>Description: TODO </li>
     *
     * @param pathFile TODO
     */
    public void loadData(String pathFile){
        String sql = "LOAD DATA INPATH  '"+pathFile+"' INTO TABLE bus_receiver";
        hiveJdbcTemplate.execute(sql);
    }


    /**
     * <li>Description: TODO </li>
     *
     * @param busReceiverEntity 实体
     */
    public void insert(BusReceiverEntity busReceiverEntity) {
        hiveJdbcTemplate.update("insert into bus_receiver(id,name,address,en_name,member_family) values(?,?,?,?,?)",
                new PreparedStatementSetter(){
                    @Override
                    public void setValues(PreparedStatement ps) throws SQLException {
                        ps.setLong(1, busReceiverEntity.getId());
                        ps.setString(2,busReceiverEntity.getName());
                        ps.setString(3,busReceiverEntity.getAddress());
                        ps.setString(4,busReceiverEntity.getEnName());
                        ps.setInt(5,busReceiverEntity.getMemberFamily());
                    }
                }
        );
    }

    public void deleteAll(){
        String sql = "insert overwrite table bus_receiver select * from bus_receiver where 1=0";
        hiveJdbcTemplate.execute(sql);
    }
}
最后贴上配置文件:

hive:
  url: jdbc:hive2://XXXX-slave2.phmcluster.calabar:10000/test
  driver-class-name: org.apache.hive.jdbc.HiveDriver
  user:
  password:

需要注意的是,再启动项目的时候需要将servlet-api放到JAVA_HOME/jre/lib/ext目录下

以上方案并非完全原创.