http://blog.chinaunix.net/u3/102568/article_144792.html
http://blog.csdn.net/dajuezhao/category/724896.aspx
http://www.javabloger.com/article/apache-hbase-shell-and-install-key-value.html
以上三个里面的所有资料都看了,相信你就知道一定的hbase概念了。
好了,现在讲讲我的配置环境:
cygwin + hadoop-0.20.2 + zookeeper-3.3.2 + hbase-0.20.6 (+ eclipse3.6)
具体的配置细节,这里不讲了,网上很多,只要细心就没问题。
假设都配置好了,那么启动这些服务吧,据说启动顺序也是有要求的:
1,hadoop ./start-all.sh
2,zookeeper ./zkServer.sh start
3,hbase ./start-hbase.sh
停止的时候也是有顺序的, hbase--zookeeper--hadoop
成功后的界面截图:
http://localhost:60010/master.jsp 【hbase的管理信息】
下面就写java代码来操作hbase,我写了简单的增删改查:
package org.test; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Scanner; import org.apache.hadoop.hbase.io.BatchUpdate; import org.apache.hadoop.hbase.io.Cell; import org.apache.hadoop.hbase.io.RowResult; import org.apache.hadoop.hbase.util.Bytes; public class TestHBase { public final static String COLENDCHAR = String.valueOf(KeyValue.COLUMN_FAMILY_DELIMITER);//":" final String key_colName = "colN"; final String key_colCluster = "colClut"; final String key_colDataType = "colDT"; final String key_colVal = "colV"; //hbase的环境变量 HBaseConfiguration conf; HBaseAdmin admin = null; /** * @param args */ public static void main(String[] args) { // TODO Auto-generated method stub TestHBase app = new TestHBase(); //app.test(); app.init(); app.go(); app.list(); } void list(){ try { String tableName = "htcjd0"; Map rsMap = this.getHTData(tableName); System.out.println(rsMap.toString()); } catch (Exception e) { e.printStackTrace(); } } void go(){ try { //建表 String tableName = "htcjd0"; String[] columns = new String[]{"col"}; this.createHTable(tableName, columns); //插入数据 List list = new ArrayList(); List rowList = null; Map rowMap = null; for (int i = 0; i < 10; i++) { rowList = new ArrayList(); rowMap = new HashMap(); rowMap.put(key_colName, "col"); //rowMap.put(key_colCluster, "cl_name"); rowMap.put(key_colVal, "陈杰堆nocluster"+i); rowList.add(rowMap); rowMap = new HashMap(); rowMap.put(key_colName, "col"); rowMap.put(key_colCluster, "cl_name"); rowMap.put(key_colVal, "陈杰堆cl_"+i); rowList.add(rowMap); rowMap = new HashMap(); rowMap.put(key_colName, "col"); rowMap.put(key_colCluster, "cl_age"); rowMap.put(key_colVal, "cl_"+i); rowList.add(rowMap); rowMap = new HashMap(); rowMap.put(key_colName, "col"); rowMap.put(key_colCluster, "cl_sex"); rowMap.put(key_colVal, "列cl_"+i); rowList.add(rowMap); list.add(rowList); } HTable hTable = this.getHTable(tableName); this.insertRow(hTable, list); } catch (Exception e) { e.printStackTrace(); } } void go0(){ try { //建表 String tableName = "htcjd"; String[] columns = new String[]{"name","age","col"}; this.createHTable(tableName, columns); //插入数据 List list = new ArrayList(); List rowList = null; Map rowMap = null; for (int i = 0; i < 10; i++) { rowList = new ArrayList(); rowMap = new HashMap(); rowMap.put(key_colName, "name"); rowMap.put(key_colVal, "测试hbase"+i); rowMap.put(key_colName, "age"); rowMap.put(key_colVal, ""+i); rowMap.put(key_colName, "col"); rowMap.put(key_colVal, "列"+i); rowList.add(rowMap); list.add(rowList); } HTable hTable = this.getHTable(tableName); this.insertRow(hTable, list); } catch (Exception e) { e.printStackTrace(); } } void init() { try { Configuration HBASE_CONFIG = new Configuration(); HBASE_CONFIG.set("hbase.zookeeper.quorum", "127.0.0.1"); HBASE_CONFIG.set("hbase.zookeeper.property.clientPort", "2181"); this.conf = new HBaseConfiguration(HBASE_CONFIG);// new HBaseConfiguration(); this.admin = new HBaseAdmin(conf); } catch (Exception e) { e.printStackTrace(); } } /** * 创建表的描述 * @param tableName * @return * @throws Exception */ HTableDescriptor createHTDesc(final String tableName)throws Exception{ try { return new HTableDescriptor(tableName); } catch (Exception e) { throw e; } } /** * 针对hbase的列的特殊情况进行处理,列的情况: course: or course:math, * 就是要么带列族,要么不带列族(以冒号结尾) * @param colName 列 * @param cluster 列族 * @return */ String fixColName(String colName,String cluster){ if(cluster!=null&&cluster.trim().length()>0&&colName.endsWith(cluster)){ return colName; } String tmp = colName; int index = colName.indexOf(COLENDCHAR); //int leng = colName.length(); if(index == -1){ tmp += COLENDCHAR; } //直接加入列族 if(cluster!=null&&cluster.trim().length()>0){ tmp += cluster; } return tmp; } String fixColName(String colName){ return this.fixColName(colName, null); } /** * 创建列的描述,添加后,该列会有一个冒号的后缀,用于存储(列)族, * 将来如果需要扩展,那么就在该列后加入(列)族 * @param colName * @return * @throws Exception */ HColumnDescriptor createHCDesc(String colName)throws Exception{ try { String tmp = this.fixColName(colName); byte[] colNameByte = Bytes.toBytes(tmp); return new HColumnDescriptor(colNameByte); } catch (Exception e) { throw e; } } /** * 给表添加列,此时不带列族 * @param htdesc * @param colName * @param readonly * @throws Exception */ void addFamily(HTableDescriptor htdesc,String colName,final boolean readonly)throws Exception{ try { htdesc.addFamily(this.createHCDesc(colName)); htdesc.setReadOnly(readonly); } catch (Exception e) { throw e; } } /** * 删除列--不带列族 * @param tableName * @param colName * @throws Exception */ void removeFamily(String tableName,String colName)throws Exception{ try { String tmp = this.fixColName(colName); this.admin.deleteColumn(tableName, tmp); } catch (Exception e) { throw e; } } /** * 删除列--带列族 * @param tableName * @param colName * @param cluster * @throws Exception */ void removeFamily(String tableName,String colName,String cluster)throws Exception{ try { String tmp = this.fixColName(colName,cluster); this.admin.deleteColumn(tableName, tmp); } catch (Exception e) { throw e; } } /** * 建表 * @param tableName * @param columns * @throws Exception */ void createHTable(String tableName)throws Exception{ try { if(admin.tableExists(tableName))return;//判断表是否已经存在 HTableDescriptor htdesc = this.createHTDesc(tableName); admin.createTable(htdesc); } catch (Exception e) { throw e; } } void createHTable(String tableName,String[] columns)throws Exception{ try { if(admin.tableExists(tableName))return;//判断表是否已经存在 HTableDescriptor htdesc = this.createHTDesc(tableName); for (int i = 0; i < columns.length; i++) { String colName = columns[i]; this.addFamily(htdesc, colName, false); } admin.createTable(htdesc); } catch (Exception e) { throw e; } } /** * 删除表 * @param tableName * @throws Exception */ void removeHTable(String tableName)throws Exception{ try { admin.disableTable(tableName);//使无效 admin.deleteTable(tableName);//再删除 } catch (Exception e) { throw e; } } /** * 取得某个表 * @param tableName * @return * @throws Exception */ HTable getHTable(String tableName)throws Exception{ try { return new HTable(conf, tableName); } catch (Exception e) { throw e; } } void updateColumn(String tableName,String rowID,String colName,String cluster,String value)throws Exception{ try { BatchUpdate batchUpdate = new BatchUpdate(rowID); String tmp = this.fixColName(colName, cluster); batchUpdate.put(tmp, Bytes.toBytes(value)); HTable hTable = this.getHTable(tableName); hTable.commit(batchUpdate); } catch (Exception e) { throw e; } } void updateColumn(String tableName,String rowID,String colName,String value)throws Exception{ try { this.updateColumn(tableName, rowID, colName, null, value); } catch (Exception e) { throw e; } } void deleteColumn(String tableName,String rowID,String colName,String cluster)throws Exception{ try { BatchUpdate batchUpdate = new BatchUpdate(rowID); String tmp = this.fixColName(colName, cluster); batchUpdate.delete(tmp); HTable hTable = this.getHTable(tableName); hTable.commit(batchUpdate); } catch (Exception e) { throw e; } } void deleteColumn(String tableName,String rowID,String colName)throws Exception{ try { this.deleteColumn(tableName, rowID, colName, null); } catch (Exception e) { throw e; } } /** * 取得某一行,某一列的值 * @param tableName * @param rowID * @param colName * @param cluster * @return * @throws Exception */ String getColumnValue(String tableName,String rowID,String colName,String cluster)throws Exception{ try { String tmp = this.fixColName(colName, cluster); HTable hTable = this.getHTable(tableName); Cell cell = hTable.get(rowID, tmp); if(cell==null)return null; return new String(cell.getValue()); } catch (Exception e) { throw e; } } /** * 取得所属列的值 * @param tableName * @param colName * @param cluster 如果该参数为空,那么返回所有列族的值 * @return * @throws Exception */ Map getColumnValue(String tableName, String colName, String cluster)throws Exception { Scanner scanner = null; try { String tmp = this.fixColName(colName, cluster); HTable hTable = this.getHTable(tableName); scanner = hTable.getScanner(new String[] { tmp });// "myColumnFamily:columnQualifier1" RowResult rowResult = scanner.next(); Map resultMap = new HashMap(); String row, value; Cell cell = null; while (rowResult != null) { // print out the row we found and the columns we were looking // for // System.out.println("Found row: " // + new String(rowResult.getRow()) // + " with value: " // + rowResult.get("myColumnFamily:columnQualifier1" // .getBytes())); row = new String(rowResult.getRow()); cell = rowResult.get(Bytes.toBytes(tmp)); if (cell == null) { resultMap.put(row, null); } else { resultMap.put(row, cell.getValue()); } rowResult = scanner.next(); } return resultMap; } catch (Exception e) { throw e; }finally{ if(scanner!=null){ scanner.close();//一定要关闭 } } } /** * 取得所有数据 * @param tableName * @return Map * @throws Exception */ public Map getHTData(String tableName) throws Exception { ResultScanner rs = null; try { HTable table = new HTable(this.conf, tableName); Scan s = new Scan(); rs = table.getScanner(s); Map resultMap = new HashMap(); for (Result r : rs) { for (KeyValue kv : r.raw()) { resultMap.put(new String(kv.getColumn()), new String(kv.getValue())); } } return resultMap; } catch (Exception e) { throw e; } finally { if (rs != null) rs.close(); } } //插入记录 void insertRow(HTable table,List dataList)throws Exception{ try { Put put = null; String colName = null; String colCluster = null; String colDataType = null; byte[] value; List rowDataList = null; Map rowDataMap = null; for (Iterator iterator = dataList.iterator(); iterator.hasNext();) { rowDataList = (List) iterator.next(); for(int i=0;i<rowDataList.size();i++){ rowDataMap = (Map) rowDataList.get(i); colName = (String)rowDataMap.get(key_colName); colCluster = (String)rowDataMap.get(key_colCluster); colDataType = (String)rowDataMap.get(key_colDataType); Object val = rowDataMap.get(key_colVal); value = Bytes.toBytes(String.valueOf(val)); // //根据数据类型来处理 // if("string".equalsIgnoreCase(colDataType)){ // value = Bytes.toBytes((String)val); // }else if("int".equalsIgnoreCase(colDataType)){ // value = Bytes.toInt(Integer.parseInt(String.valueOf(val))); // }else if("float".equalsIgnoreCase(colDataType)){ // value = Bytes.toBytes(Float.parseFloat(String.valueOf(val))); // }else if("long".equalsIgnoreCase(colDataType)){ // value = Bytes.toBytes(Long.parseLong(String.valueOf(val))); // }else if("double".equalsIgnoreCase(colDataType)){ // value = Bytes.toBytes(Double.parseDouble(String.valueOf(val))); // }else if("char".equalsIgnoreCase(colDataType)){ // value = Bytes.toBytes((String.valueOf(val))); // }else if("byte".equalsIgnoreCase(colDataType)){ // value = Bytes.totoBytes((byte[])val); // } put = new Put(value); String tmp = this.fixColName(colName, colCluster); byte[] colNameByte = Bytes.toBytes(tmp); byte[][] famAndQf = KeyValue.parseColumn(colNameByte); put.add(famAndQf[0], famAndQf[1], value); table.put(put); } } } catch (Exception e) { throw e; } } //取得表的结构信息 }
然后在eclipse里面运行,可以看到结果:
[hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:zookeeper.version=3.3.2-1031432, built on 11/05/2010 05:32 GMT [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:host.name=chenjiedui [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:java.version=1.6.0_05 [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:java.vendor=Sun Microsystems Inc. [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:java.home=D:\jdk1.6.0_05\jre [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:java.class.path=D:\workspace\MyHadoopApp\bin;D:\workspace\MyHadoopApp\lib\commons-lang-2.4.jar;D:\workspace\MyHadoopApp\lib\commons-logging-1.1.1.jar;D:\workspace\Hadoop0.20.2\bin;D:\workspace\Hadoop0.20.2\lib\commons-cli-1.2.jar;D:\workspace\Hadoop0.20.2\lib\commons-codec-1.3.jar;D:\workspace\Hadoop0.20.2\lib\commons-el-1.0.jar;D:\workspace\Hadoop0.20.2\lib\commons-httpclient-3.0.1.jar;D:\workspace\Hadoop0.20.2\lib\commons-logging-1.0.4.jar;D:\workspace\Hadoop0.20.2\lib\commons-logging-api-1.0.4.jar;D:\workspace\Hadoop0.20.2\lib\commons-net-1.4.1.jar;D:\workspace\Hadoop0.20.2\lib\core-3.1.1.jar;D:\workspace\Hadoop0.20.2\lib\hsqldb-1.8.0.10.jar;D:\workspace\Hadoop0.20.2\lib\jasper-compiler-5.5.12.jar;D:\workspace\Hadoop0.20.2\lib\jasper-runtime-5.5.12.jar;D:\workspace\Hadoop0.20.2\lib\jets3t-0.6.1.jar;D:\workspace\Hadoop0.20.2\lib\jetty-6.1.14.jar;D:\workspace\Hadoop0.20.2\lib\jetty-util-6.1.14.jar;D:\workspace\Hadoop0.20.2\lib\junit-3.8.1.jar;D:\workspace\Hadoop0.20.2\lib\kfs-0.2.2.jar;D:\workspace\Hadoop0.20.2\lib\log4j-1.2.15.jar;D:\workspace\Hadoop0.20.2\lib\mockito-all-1.8.0.jar;D:\workspace\Hadoop0.20.2\lib\oro-2.0.8.jar;D:\workspace\Hadoop0.20.2\lib\servlet-api-2.5-6.1.14.jar;D:\workspace\Hadoop0.20.2\lib\slf4j-api-1.4.3.jar;D:\workspace\Hadoop0.20.2\lib\slf4j-log4j12-1.4.3.jar;D:\workspace\Hadoop0.20.2\lib\xmlenc-0.52.jar;D:\workspace\Hadoop0.20.2\lib\ant.jar;D:\workspace\Hadoop0.20.2\lib\jsp-2.1.jar;D:\workspace\Hadoop0.20.2\lib\jsp-api-2.1.jar;D:\workspace\Hadoop0.20.2\lib\ftplet-api-1.0.0-SNAPSHOT.jar;D:\workspace\Hadoop0.20.2\lib\ftpserver-core-1.0.0-SNAPSHOT.jar;D:\workspace\Hadoop0.20.2\lib\ftpserver-server-1.0.0-SNAPSHOT.jar;D:\workspace\Hadoop0.20.2\lib\libthrift.jar;D:\workspace\Hadoop0.20.2\lib\mina-core-2.0.0-M2-20080407.124109-12.jar;D:\workspace\Hadoop0.20.2\libs\lucene\lucene-core-3.0.1.jar;D:\workspace\HBase0.20.6\bin;D:\workspace\HBase0.20.6\lib\commons-cli-2.0-SNAPSHOT.jar;D:\workspace\HBase0.20.6\lib\commons-el-from-jetty-5.1.4.jar;D:\workspace\HBase0.20.6\lib\commons-httpclient-3.0.1.jar;D:\workspace\HBase0.20.6\lib\commons-logging-1.0.4.jar;D:\workspace\HBase0.20.6\lib\commons-logging-api-1.0.4.jar;D:\workspace\HBase0.20.6\lib\commons-math-1.1.jar;D:\workspace\HBase0.20.6\lib\hadoop-0.20.2-core.jar;D:\workspace\HBase0.20.6\lib\jasper-compiler-5.5.12.jar;D:\workspace\HBase0.20.6\lib\jasper-runtime-5.5.12.jar;D:\workspace\HBase0.20.6\lib\jetty-6.1.14.jar;D:\workspace\HBase0.20.6\lib\jetty-util-6.1.14.jar;D:\workspace\HBase0.20.6\lib\jruby-complete-1.2.0.jar;D:\workspace\HBase0.20.6\lib\junit-4.8.1.jar;D:\workspace\HBase0.20.6\lib\libthrift-r771587.jar;D:\workspace\HBase0.20.6\lib\log4j-1.2.15.jar;D:\workspace\HBase0.20.6\lib\lucene-core-2.2.0.jar;D:\workspace\HBase0.20.6\lib\servlet-api-2.5-6.1.14.jar;D:\workspace\HBase0.20.6\lib\xmlenc-0.52.jar;D:\workspace\HBase0.20.6\lib\zookeeper-3.3.2.jar;D:\workspace\MyHadoopApp\lib\commons-cli-2.0-SNAPSHOT.jar;D:\workspace\MyHadoopApp\lib\log4j-1.2.15.jar;D:\workspace\MyHadoopApp\lib\hbase\commons-el-from-jetty-5.1.4.jar;D:\workspace\MyHadoopApp\lib\hbase\commons-httpclient-3.0.1.jar;D:\workspace\MyHadoopApp\lib\hbase\commons-logging-api-1.0.4.jar;D:\workspace\MyHadoopApp\lib\hbase\commons-math-1.1.jar;D:\workspace\MyHadoopApp\lib\hbase\jasper-compiler-5.5.12.jar;D:\workspace\MyHadoopApp\lib\hbase\jasper-runtime-5.5.12.jar;D:\workspace\MyHadoopApp\lib\hbase\jetty-6.1.14.jar;D:\workspace\MyHadoopApp\lib\hbase\jetty-util-6.1.14.jar;D:\workspace\MyHadoopApp\lib\hbase\jruby-complete-1.2.0.jar;D:\workspace\MyHadoopApp\lib\hbase\libthrift-r771587.jar;D:\workspace\MyHadoopApp\lib\hbase\lucene-core-2.2.0.jar;D:\workspace\MyHadoopApp\lib\hbase\servlet-api-2.5-6.1.14.jar;D:\workspace\MyHadoopApp\lib\hbase\xmlenc-0.52.jar;D:\workspace\MyHadoopApp\lib\hbase\zookeeper-3.3.2.jar [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:java.library.path=D:\jdk1.6.0_05\bin;.;C:\WINDOWS\Sun\Java\bin;C:\WINDOWS\system32;C:\WINDOWS;d:/jdk1.6.0_05/bin/../jre/bin/client;d:/jdk1.6.0_05/bin/../jre/bin;d:/jdk1.6.0_05/bin/../jre/lib/i386;D:\cygwin\bin;D:\cygwin\usr\sbin;d:\oracle\product\10.2.0\db_1\bin;d:\jdk1.6.0_05\bin;D:\apache-ant-1.8.0RC1\bin;C:\WINDOWS\system32;C:\WINDOWS;C:\WINDOWS\System32\Wbem;C:\Program Files\Intel\WiFi\bin\;C:\Program Files\ThinkPad\ConnectUtilities;C:\Program Files\Common Files\Lenovo;d:\Program Files\cvsnt;C:\Program Files\Common Files\Thunder Network\KanKan\Codecs;C:\Program Files\Common Files\TTKN\Bin;C:\Program Files\StormII\Codec;C:\Program Files\StormII [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:java.io.tmpdir=C:\DOCUME~1\ADMINI~1\LOCALS~1\Temp\ [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:java.compiler=<NA> [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:os.name=Windows XP [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:os.arch=x86 [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:os.version=5.1 [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:user.name=Administrator [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:user.home=C:\Documents and Settings\Administrator [hadoop] INFO [main] ZooKeeper.logEnv(97) | Client environment:user.dir=D:\workspace\MyHadoopApp [hadoop] INFO [main] ZooKeeper.<init>(373) | Initiating client connection, connectString=127.0.0.1:2181 sessionTimeout=60000 watcher=org.apache.hadoop.hbase.client.HConnectionManager$ClientZKWatcher@cd2c3c [hadoop] INFO [main-SendThread()] ClientCnxn.startConnect(1041) | Opening socket connection to server /127.0.0.1:2181 [hadoop] INFO [main-SendThread(localhost:2181)] ClientCnxn.primeConnection(949) | Socket connection established to localhost/127.0.0.1:2181, initiating session [hadoop] INFO [main-SendThread(localhost:2181)] ClientCnxn.readConnectResult(738) | Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x12c6c8d8f6d0010, negotiated timeout = 40000 输出结果: {col:cl_name=陈杰堆cl_9, col:name=陈杰堆9, col:sex=列9, col:cl_age=cl_9, col:cl_sex=列cl_9, col:age=9, col:=陈杰堆nocluster9} [hadoop] INFO [HCM.shutdownHook] ZooKeeper.close(538) | Session: 0x12c6c8d8f6d0010 closed [hadoop] INFO [main-EventThread] ClientCnxn.run(520) | EventThread shut down