由于公司项目需要存储大量的图片和pdf文件,采用了hbase数据库进行存储,现已将hadoop集群(三个节点)和zookeeper搭建完成,下面主要介绍如何使用Java连接zookeeper实现hbase的文件存储和查询。
- 添加pom文件依赖
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.1.2</version>
<exclusions>
<exclusion>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</exclusion>
</exclusions>
</dependency>
- yml文件配置端口和地址
hbaseClient:
zookeeperPort: 2181
zookeeperQuorum: 192.168.2.14,192.168.2.15,192.168.2.16
- 注入类
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
@Component
public class ZkContant {
public static String zookeeperPort;
public static String zookeeperQuorum;
public String getZookeeperPort() {
return zookeeperPort;
}
@Value("${hbaseClient.zookeeperPort}")
public void setZookeeperPort(String zookeeperPort) {
ZookeeperContant.zookeeperPort = zookeeperPort;
}
public String getZookeeperQuorum() {
return zookeeperQuorum;
}
@Value("${hbaseClient.zookeeperQuorum}")
public void setZookeeperQuorum(String zookeeperQuorum) {
ZookeeperContant.zookeeperQuorum = zookeeperQuorum;
}
}
- hbase操作工具类
import com.gpxx.airchina.constant.ZookeeperContant;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
public class HbaseUtil {
private static final Logger log = LogManager.getLogger(HbaseUtil.class);
private static Configuration configuration = null;
private static Connection con = null;
static {
try {
String clan_clientPort =ZookeeperContant.zookeeperPort;
String clan_quorum = ZookeeperContant.zookeeperQuorum;
String clan_master = "";
configuration = HBaseConfiguration.create();
configuration.set("hbase.zookeeper.property.clientPort", clan_clientPort);
configuration.set("hbase.zookeeper.quorum", clan_quorum);// 设置zookeeper的主机
if (clan_master != null && !"".equals(clan_master))
configuration.set("hbase.master", clan_master);// 设置hbase的master主机名和端口
con = ConnectionFactory.createConnection(configuration);
} catch (Exception ex) {
log.error("初始化异常");
ex.printStackTrace();
}
}
/**
* 保存
*/
public static void insertHbaseData(String rowKey, String val) throws Exception {
Table table = null;
String colName = "DATA";//列簇
String tableName = "file_image_save";//表名
try {
table = con.getTable(TableName.valueOf(tableName));
Put put = new Put(Bytes.toBytes(rowKey));
put.addColumn(Bytes.toBytes(colName), Bytes.toBytes(rowKey), Bytes.toBytes(val));
table.put(put);
} finally {
if (table != null)
table.close();
}
}
/**
*查询
*/
public static String getData(String rowkey) throws IOException {
Map returnMap = new HashMap();
String colName = "DATA";
String tableName = "file_image_save";
Table table = null;
try {
table = con.getTable(TableName.valueOf(tableName));
Get get = new Get(Bytes.toBytes(rowkey));
get.addColumn(Bytes.toBytes(colName), Bytes.toBytes(rowkey));
Result r = table.get(get);
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
returnMap.put(Bytes.toString(CellUtil.cloneQualifier(cell)), Bytes.toString(CellUtil.cloneValue(cell)));
}
} finally {
if (table != null)
table.close();
}
return returnMap.get(rowkey)==null?null:returnMap.get(rowkey).toString();
}
}
- controller业务层的实现
@PostMapping("/uploadImg.do")
public ResultResponse uploadIMG(@RequestParam("img") MultipartFile img,
@RequestParam("id") String id
) {
long MAX_SIZE = 3 * 1024 * 1024;// 设置上传文件最大为3M
long fileSize = img.getSize(); //文件大小
String originalFilename = img.getOriginalFilename();//文件名称
String fileType = FilenameUtils.getExtension(originalFilename).toLowerCase(); //文件类型
if(!".png|.jpg|.jpeg|.bmp".contains(fileType)){
response.setMessage("文件类型不正确,请上传图片文件");
return response;
}
if(fileSize > MAX_SIZE){
response.setMessage("单文件不能超过3M");
return response;
}
ResultResponse response = new ResultResponse();
try {
if (img != null && img.getSize()>0) {
//上传文件
InputStream is = img.getInputStream();
byte[] b = new byte[(int) img.getSize()];
is.read(b);
String imgbase64=BASE64Util.encode(b);//存储为二进制文件
is.close();
HbaseUtil.insertHbaseData(id,imgbase64);//id为rowkey
response.setMessage("上传成功");
}else {
response.setMessage("请选择文件上传");
}
} catch (Exception e) {
e.printStackTrace();
response.setMessage("上传失败"+e.getMessage());
}
return response;
}
@PostMapping("/loadImg.do")
public ResultResponse loadIMG(@RequestParam("id") String id){
ResultResponse response = new ResultResponse();
List<Object> fileList = new ArrayList();
try {
String imgData = HbaseUtil.getData(id);
fileList.add(BASE64Util.base64Decoder(imgData));
} catch (IOException e) {
e.printStackTrace();
response.setMessage("查询异常:"+e.getMessage());
return response;
}catch (Exception e) {
e.printStackTrace();
response.setMessage("查询异常:"+e.getMessage());
return response;
}
response.setMessage("查询成功");
response.setData(fileList);
return response;
}