Hbase 过滤器

其他 2019-04-23 15:40:53 阅读次数: 0 
package com.mao.hbase;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CompareOperator;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.filter.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;

import java.io.IOException;
import java.util.Iterator;

public class HbaseFilteTest {

    private Connection hbaseConn() throws IOException {
        //创建conf对象
        Configuration conf = HBaseConfiguration.create();
        //通过连接工厂创建连接对象
        return ConnectionFactory.createConnection(conf);
    }

    /**
     * 行过滤器
     * select * from ns1:t1 where rowkey <= row100
     * @throws IOException
     */
    @Test
    public void testRowFilter() throws IOException {

        TableName tableName = TableName.valueOf("ns1:t1");
        Scan scan = new Scan();
        RowFilter rowFilter = new RowFilter(CompareOperator.LESS_OR_EQUAL,new BinaryComparator(Bytes.toBytes("row1000")));
        scan.setFilter(rowFilter);
        Table t = hbaseConn().getTable(tableName);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()){
            Result r = it.next();
            System.out.println(Bytes.toString(r.getRow()));
        }
    }

    /**
     * 列族过滤器
     * @throws IOException
     */
    @Test
    public void testFamilyFilter() throws IOException {

        TableName tableName = TableName.valueOf("ns1:t7");
        Scan scan = new Scan();
        FamilyFilter filter = new FamilyFilter(CompareOperator.LESS,new BinaryComparator(Bytes.toBytes("f2")));
        scan.setFilter(filter);
        Table t = hbaseConn().getTable(tableName);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()){
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("id"));
            byte[] f2id = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("id"));
            System.out.println("f1id: "+Bytes.toString(f1id)+",  f2id:"+Bytes.toString(f2id));
        }
    }

    /**
     * 列过滤器
     * @throws IOException
     */
    @Test
    public void testQualifierFilter() throws IOException {

        TableName tableName = TableName.valueOf("ns1:t7");
        Scan scan = new Scan();
        QualifierFilter filter = new QualifierFilter(CompareOperator.EQUAL,new BinaryComparator(Bytes.toBytes("id")));
        scan.setFilter(filter);
        Table t = hbaseConn().getTable(tableName);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()){
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("id"));
            byte[] f2id = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("id"));
            byte[] f2name = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("name"));
            System.out.println("f1id: "+Bytes.toString(f1id)+",  f2id:"+Bytes.toString(f2id)+", f2name:"+Bytes.toString(f2name));
        }
    }

    /**
     * 值过滤器
     * 过滤value的值，含有指定的字符子串
     * @throws IOException
     */
    @Test
    public void testValueFilter() throws IOException {

        TableName tableName = TableName.valueOf("ns1:t7");
        Scan scan = new Scan();
        ValueFilter filter = new ValueFilter(CompareOperator.EQUAL,new SubstringComparator(".1"));
        scan.setFilter(filter);
        Table t = hbaseConn().getTable(tableName);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()){
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("id"));
            byte[] f1name = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("name"));
            byte[] f2id = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("id"));
            byte[] f2name = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("name"));
            System.out.println("f1id: "+Bytes.toString(f1id)+", f1name:"+Bytes.toString(f1name)
                    +",  f2id:"+Bytes.toString(f2id)+", f2name:"+Bytes.toString(f2name));
        }
    }

    /**
     * 依赖列过滤器
     *
     * @throws IOException
     */
    @Test
    public void testDepFilter() throws IOException {

        TableName tableName = TableName.valueOf("ns1:t7");
        Scan scan = new Scan();
        DependentColumnFilter filter = new DependentColumnFilter(Bytes.toBytes("f2"),
                Bytes.toBytes("name"),
                false,
                CompareFilter.CompareOp.NOT_EQUAL,
                new BinaryComparator(Bytes.toBytes("tom2.1")));

        scan.setFilter(filter);
        Table t = hbaseConn().getTable(tableName);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()){
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("id"));
            byte[] f1name = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("name"));
            byte[] f2id = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("id"));
            byte[] f2name = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("name"));
            System.out.println("f1id: "+Bytes.toString(f1id)+", f1name:"+Bytes.toString(f1name)
                    +",  f2id:"+Bytes.toString(f2id)+", f2name:"+Bytes.toString(f2name));
        }
    }

    /**
     * 单列值过滤器
     * 对列上的value进行过滤，不符合整行删除
     * @throws IOException
     */
    @Test
    public void testSingleColumValueFilter() throws IOException {

        TableName tableName = TableName.valueOf("ns1:t7");
        Scan scan = new Scan();
        SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("f2"),
                Bytes.toBytes("name"),
                CompareFilter.CompareOp.NOT_EQUAL,
                new BinaryComparator(Bytes.toBytes("tom2.1")));

        scan.setFilter(filter);
        Table t = hbaseConn().getTable(tableName);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()){
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("id"));
            byte[] f1name = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("name"));
            byte[] f2id = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("id"));
            byte[] f2name = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("name"));
            System.out.println("f1id: "+Bytes.toString(f1id)+", f1name:"+Bytes.toString(f1name)
                    +",  f2id:"+Bytes.toString(f2id)+", f2name:"+Bytes.toString(f2name));
        }
    }

    /**
     * 单列值排除过滤器,去掉过滤使用的列,对列的值进行过滤
     */
    @Test
    public void testSingleColumnValueExcludeFilter() throws IOException {

        TableName tableName = TableName.valueOf("ns1:t7");
        Scan scan = new Scan();
        SingleColumnValueExcludeFilter filter = new SingleColumnValueExcludeFilter(Bytes.toBytes("f1"),
                Bytes.toBytes("name"),
                CompareFilter.CompareOp.EQUAL,
                new BinaryComparator(Bytes.toBytes("tom2.1")));

        scan.setFilter(filter);
        Table t = hbaseConn().getTable(tableName);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()){
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("id"));
            byte[] f1name = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("name"));
            byte[] f2id = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("id"));
            byte[] f2name = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("name"));
            System.out.println("f1id: "+Bytes.toString(f1id)+", f1name:"+Bytes.toString(f1name)
                    +",  f2id:"+Bytes.toString(f2id)+", f2name:"+Bytes.toString(f2name));
        }
    }

    /**
     * 前缀过滤器
     * 是rowkey过滤. where rowkey like 'row22%'
     * @throws IOException
     */
    @Test
    public void testPrefixFilter() throws IOException {

        TableName tableName = TableName.valueOf("ns1:t1");
        Scan scan = new Scan();
        PrefixFilter rowFilter = new PrefixFilter(Bytes.toBytes("row222"));
        scan.setFilter(rowFilter);
        Table t = hbaseConn().getTable(tableName);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()){
            Result r = it.next();
            System.out.println(Bytes.toString(r.getRow()));
        }
    }
    /**
     * 分页过滤,是rowkey过滤,在region上扫描时，对每次page设置的大小。
     * 返回到到client，设计到每个Region结果的合并。
     */
    @Test
    public void testPageFilter() throws IOException {

        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        TableName tname = TableName.valueOf("ns1:t1");
        Scan scan = new Scan();

        PageFilter filter = new PageFilter(10);

        scan.setFilter(filter);
        Table t = conn.getTable(tname);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()){
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("id"));
            byte[] f1name = r.getValue(Bytes.toBytes("f1"),Bytes.toBytes("name"));
            byte[] f2id = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("id"));
            byte[] f2name = r.getValue(Bytes.toBytes("f2"),Bytes.toBytes("name"));
            System.out.println("f1id: "+Bytes.toString(f1id)+", f1name:"+Bytes.toString(f1name)
                    +",  f2id:"+Bytes.toString(f2id)+", f2name:"+Bytes.toString(f2name));
        }
    }

    /**
     * keyOnly过滤器，只提取key,丢弃value.
     * @throws IOException
     */
    @Test
    public void testKeyOnlyFilter() throws IOException {

        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        TableName tname = TableName.valueOf("ns1:t1");
        Scan scan = new Scan();

        KeyOnlyFilter filter = new KeyOnlyFilter();

        scan.setFilter(filter);
        Table t = conn.getTable(tname);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()) {
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"), Bytes.toBytes("id"));
            byte[] f2id = r.getValue(Bytes.toBytes("f2"), Bytes.toBytes("id"));
            byte[] f1name = r.getValue(Bytes.toBytes("f1"), Bytes.toBytes("name"));
            byte[] f2name = r.getValue(Bytes.toBytes("f2"), Bytes.toBytes("name"));
            System.out.println("f1id: "+Bytes.toString(f1id)+", f1name:"+Bytes.toString(f1name)
                    +",  f2id:"+Bytes.toString(f2id)+", f2name:"+Bytes.toString(f2name));
        }
    }

    /**
     * ColumnPageFilter,列分页过滤器，过滤指定范围列，
     * select ,,a,b from ns1:t7
     */
    @Test
    public void testColumnPageFilter() throws IOException {

        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        TableName tname = TableName.valueOf("ns1:t7");
        Scan scan = new Scan();

        ColumnPaginationFilter filter = new ColumnPaginationFilter(2,2);

        scan.setFilter(filter);
        Table t = conn.getTable(tname);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()) {
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"), Bytes.toBytes("id"));
            byte[] f2id = r.getValue(Bytes.toBytes("f2"), Bytes.toBytes("id"));
            byte[] f1name = r.getValue(Bytes.toBytes("f1"), Bytes.toBytes("name"));
            byte[] f2name = r.getValue(Bytes.toBytes("f2"), Bytes.toBytes("name"));
            System.out.println("f1id: "+Bytes.toString(f1id)+", f1name:"+Bytes.toString(f1name)
                    +",  f2id:"+Bytes.toString(f2id)+", f2name:"+Bytes.toString(f2name));
        }
    }

    /**
     * 正则测试
     */
    @Test
    public void testLike() throws IOException {

        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        TableName tname = TableName.valueOf("ns1:t7");
        Scan scan = new Scan();

        ValueFilter filter = new ValueFilter(CompareFilter.CompareOp.EQUAL,
                new RegexStringComparator("^tom")
        );

        scan.setFilter(filter);
        Table t = conn.getTable(tname);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()) {
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"), Bytes.toBytes("id"));
            byte[] f2id = r.getValue(Bytes.toBytes("f2"), Bytes.toBytes("id"));
            byte[] f1name = r.getValue(Bytes.toBytes("f1"), Bytes.toBytes("name"));
            byte[] f2name = r.getValue(Bytes.toBytes("f2"), Bytes.toBytes("name"));
            System.out.println("f1id: "+Bytes.toString(f1id)+", f1name:"+Bytes.toString(f1name)
                    +",  f2id:"+Bytes.toString(f2id)+", f2name:"+Bytes.toString(f2name));
        }
    }

    /**
     * 复杂查询
     *select * from t7 where ((age <= 13) and (name like '%tom') or (age >= 14) and (name like 't%'))
     */
    @Test
    public void testComboFilter() throws IOException {
        TableName tname = TableName.valueOf("ns1:t7");
        Scan scan = new Scan();
        //age<=13
        SingleColumnValueFilter filter1 = new SingleColumnValueFilter(Bytes.toBytes("f2"),
                Bytes.toBytes("age"),
                CompareFilter.CompareOp.LESS_OR_EQUAL,
                new BinaryComparator(Bytes.toBytes("13")));
        //name like '%tom'
        SingleColumnValueFilter filter2 = new SingleColumnValueFilter(Bytes.toBytes("f2"),
                Bytes.toBytes("name"),
                CompareFilter.CompareOp.LESS_OR_EQUAL,
                new RegexStringComparator("^tom"));

        FilterList fa = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        fa.addFilter(filter1);
        fa.addFilter(filter2);

        //age<=13
        SingleColumnValueFilter ft1 = new SingleColumnValueFilter(Bytes.toBytes("f2"),
                Bytes.toBytes("age"),
                CompareFilter.CompareOp.GREATER_OR_EQUAL,
                new BinaryComparator(Bytes.toBytes("14")));
        //name like '%tom'
        SingleColumnValueFilter ft2 = new SingleColumnValueFilter(Bytes.toBytes("f2"),
                Bytes.toBytes("name"),
                CompareFilter.CompareOp.LESS_OR_EQUAL,
                new RegexStringComparator("tom$"));

        FilterList fb = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        fb.addFilter(filter1);
        fb.addFilter(filter2);

        FilterList filterAll = new FilterList(FilterList.Operator.MUST_PASS_ONE);
        filterAll.addFilter(fa);
        filterAll.addFilter(fb);
        scan.setFilter(filterAll);
        Table t = hbaseConn().getTable(tname);
        ResultScanner rs = t.getScanner(scan);
        Iterator<Result> it = rs.iterator();
        while (it.hasNext()) {
            Result r = it.next();
            byte[] f1id = r.getValue(Bytes.toBytes("f1"), Bytes.toBytes("id"));
            byte[] f1age = r.getValue(Bytes.toBytes("f1"), Bytes.toBytes("age"));
            byte[] f1name = r.getValue(Bytes.toBytes("f1"), Bytes.toBytes("name"));

            byte[] f2id = r.getValue(Bytes.toBytes("f2"), Bytes.toBytes("id"));
            byte[] f2age = r.getValue(Bytes.toBytes("f2"), Bytes.toBytes("age"));
            byte[] f2name = r.getValue(Bytes.toBytes("f2"), Bytes.toBytes("name"));
            System.out.println("f1id: "+Bytes.toString(f1id)+", f1name:"+Bytes.toString(f1name)+", f1age:"+Bytes.toString(f1age)
                    +",  f2id:"+Bytes.toString(f2id)+", f2name:"+Bytes.toString(f2name)+", f2age:"+Bytes.toString(f2age));
        }
    }

    /**
     *测试计数器
     */
    @Test
    public void testIncr() throws IOException {

        Configuration conf = HBaseConfiguration.create();
        Connection conn = ConnectionFactory.createConnection(conf);
        TableName tname = TableName.valueOf("ns1:t8");
        Table t = conn.getTable(tname);
        Increment incr = new Increment(Bytes.toBytes("row1"));
        incr.addColumn(Bytes.toBytes("f1"),Bytes.toBytes("daily"),1);
        incr.addColumn(Bytes.toBytes("f1"),Bytes.toBytes("weekly"),10);
        incr.addColumn(Bytes.toBytes("f1"),Bytes.toBytes("monthly"),100);
        t.increment(incr);

    }
}

猜你喜欢

转载自blog.csdn.net/mao502010435/article/details/89373315
今日推荐
TIOBE 5 月榜单：Fortran “复活”进入 Top 10
GCC 14.1 发布
面壁智能发布 Eurux-8x22B 开源大模型 —— 堪称「理科状元」
开源日报 | 谷歌扶持鸿蒙上位；开源Rabbit R1；Docker加持的安卓手机；微软的焦虑和野心；海尔电器把开放平台关了
中国码农的“35岁魔咒”
蘭雅 CorelDRAW 插件 2024.5.1 国际劳动节版，免费下载
Arc Browser for Windows 1.0 正式 GA
90后程序员开发视频搬运软件、不到一年获利超 700 万，结局很刑！
周排行
Java自定义时间格式
同步整形电路
在开发中最最最常用的字符串的属性大集合
Linux 查看端口占用并杀掉
Java基础四：ArrayList
多线程之死锁就是这么简单
mysql 基础命令集
awk 命令详解
Centos6.3编译安装nginx+php步骤
OCR （Optical Character Recognition，光学字符识别）
每日归档
更多
2024-05-08(42)
2024-05-07(14)
2024-05-06(40)
2024-05-05(0)
2024-05-04(7)
2024-05-03(19)
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)
2024-04-29(40)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022