sparkstreaming 监听 kafka 代码

● 代码，kafka不会重复消费，自己会记住偏移量
即使设置了从头开始消费，也不会消费已消费的数据

package com.ws.streaming
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}

object KafkaWordCount {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setMaster("local[*]").setAppName("KafkaWordCount")

    val ssc = new StreamingContext(conf,Seconds(5))

    val zk = "hadoop-01:2181,hadoop-02:2181,hadoop-03:2181"
    val groupId = "g1"

    val topic = Map[String,Int]("ws"->1)

    //创建kafka DStream
    val kafakaDStream: ReceiverInputDStream[(String, String)] = KafkaUtils.createStream(ssc,zk,groupId,topic)

    //ReceiverInputDStream[(String, String)]是一个元组，key是写入的key，value是实际写入的数据
    val data: DStream[String] = kafakaDStream.map(_._2)

    val result: DStream[(String, Int)] = data.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)

    result.print()

    ssc.start()

    ssc.awaitTermination()
  }
}

● 启动一个kafka命令行生产者

[root@hadoop-03 kafka_2.11-0.8.2.2]# ./bin/kafka-console-producer.sh --broker-list hadoop-01:9092,hadoop-02:9092,hadoop-03:9092 -topic ws
[2018-09-15 18:09:01,454] WARN Property topic is not valid (kafka.utils.VerifiableProperties)
hello jerert i love you m^Hbb zhang wensi
yyyyy yyyyy yyyy y y y yy

sparkstreaming 监听 kafka 代码

猜你喜欢