kafka自定义分区API

        

导入pom文件

<dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients -->
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>1.0.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-streams</artifactId>
            <version>1.0.0</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <!-- java编译插件 -->
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.2</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
        </plugins>
    </build>

Producer生产者代码

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.util.Properties;

/*
用于生产数据到kafka集群
 */
public class Producer {


    //程序的入口
    public static void main(String[] args){

        //1、配置kafka集群
        Properties props = new Properties();
        //kafka服务器地址
        props.put("bootstrap.servers", "node01:9092,node02:9092,node03:9092");
        //消息确认机制
        props.put("acks", "all");
        //重试机制
        props.put("retries", 1);
        //批量发送的大小
        props.put("batch.size", 16384);
        //消息延迟
        props.put("linger.ms", 1);
        //批量的缓冲区大小
        props.put("buffer.memory", 33554432);
        //kafka数据中key  value的序列化
        props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
        //指定自定义分区类
        props.put("partitioner.class", "task.task05.ProducerPartition");

        //2、获取producer实例
        KafkaProducer<String, String> kafkaProducer = new KafkaProducer<String, String>(props);

        ProducerPartition producerPartition = new ProducerPartition();

        //3、通过实例发送数据到kafka集群
        for (int i = 0; i <300; i++) {
            //自定义分区中
             ProducerRecord producerRecord = new ProducerRecord("title","0",""+i);
             kafkaProducer.send(producerRecord);
        }
        kafkaProducer.close();
    } 
}

自定义分区代码

import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;

import java.util.Map;

/*
实现自定义分区
 */
public class ProducerPartition implements Partitioner {
    @Override
    public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
        //把生产的数据100以内的数据分发到分区0中，100-200以内的数据分发到分区1中，200-300内的数据分发到分区2中
        Integer data = Integer.valueOf(value.toString());
        if (data<100)return 0;
        if (data>=100 && data<200)return 1;
        if (data>=200 && data<300)return 2;
        else return -1;
    }

    @Override
    public void close() {

    }

    @Override
    public void configure(Map<String, ?> configs) {

    }
}

Cosmer消费者代码

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import java.util.Arrays;
import java.util.Properties;

/*
消费/读取kafka集群的数据
 */
public class Consumer {

    public static void main(String[] args){
        Properties props = new Properties();
        //指定kafka服务器
        props.put("bootstrap.servers", "node01:9092,node02:9092,node03:9092");
        //消费组
        props.put("group.id", "test");
        //以下两行代码 ---消费者自动提交offset值
        props.put("enable.auto.commit", "true");
        //设置 当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，从头开始消费
        props.put("auto.offset.reset","earliest");
        //自动提交的周期
        props.put("auto.commit.interval.ms",  "1000");
        //设置key value的序列化
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        //2、实例Consumer
        KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<String, String>(props);
        //3、设置读取的topic
        kafkaConsumer.subscribe(Arrays.asList("title"));
        while (true){
            //4、拉取数据，并输出
            //获取所有数据
            ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(1000);
            //遍历所有数据
            for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
                //获取一条数据内的信息
                System.out.println("消费的数据是"+consumerRecord.value()+"  分区是"+consumerRecord.partition());
            }
        }
    }
}