Spark Streaming整合Kafka的两种方式

版权声明：个人博客网址 https://29dch.github.io/ GitHub网址 https://github.com/29DCH，欢迎大家前来交流探讨和star+fork! 转载请注明出处! https://blog.csdn.net/CowBoySoBusy/article/details/84668137

Maven项目下的pom.xml文件加入如下依赖

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>

1.Receiver-based
详情见我以前的相关博客
先启动zookeeper
在启动Kafka
创建topic

KafkaReceiverWordCount.scala

package spark

import org.apache.spark.SparkConf
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}

/**
  * Spark Streaming对接Kafka的方式一
  */
object KafkaReceiverWordCount {

  def main(args: Array[String]): Unit = {

    if(args.length != 4) {
      System.err.println("Usage: KafkaReceiverWordCount <zkQuorum> <group> <topics> <numThreads>")
    }

    val Array(zkQuorum, group, topics, numThreads) = args

    val sparkConf = new SparkConf() //.setAppName("KafkaReceiverWordCount")
      //.setMaster("local[2]")

    val ssc = new StreamingContext(sparkConf, Seconds(5))

    val topicMap = topics.split(",").map((_, numThreads.toInt)).toMap

    // TODO... Spark Streaming如何对接Kafka
    val messages = KafkaUtils.createStream(ssc, zkQuorum, group,topicMap)

    messages.map(_._2).flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).print()

    ssc.start()
    ssc.awaitTermination()
  }
}

2.Direct Approach
No Receiver

package spark

import org.apache.spark.SparkConf
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Seconds, StreamingContext}
import kafka.serializer.StringDecoder
/**
 * Spark Streaming对接Kafka的方式二
 */
object KafkaDirectWordCount {

 def main(args: Array[String]): Unit = {

   if(args.length != 2) {
     System.err.println("Usage: KafkaDirectWordCount <brokers> <topics>")
     System.exit(1)
   }

   val Array(brokers, topics) = args

   val sparkConf = new SparkConf() //.setAppName("KafkaReceiverWordCount")
     //.setMaster("local[2]")

   val ssc = new StreamingContext(sparkConf, Seconds(5))

   val topicsSet = topics.split(",").toSet
   val kafkaParams = Map[String,String]("metadata.broker.list"-> brokers)

   val messages = KafkaUtils.createDirectStream[String,String,StringDecoder,StringDecoder](
   ssc,kafkaParams,topicsSet
   )

   messages.map(_._2).flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).print()

   ssc.start()
   ssc.awaitTermination()
 }
}