1.实现功能:
SparkStreaming处理socket源的数据,并进行wordcount的统计。
2.scala代码
package Spark
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
/**
* spark Streaming 处理socket数据
*
* 使用nc测试nc -lk 6789
*/
object NetworkWordCount {
def main(args: Array[String]): Unit = {
val sparkConf=new SparkConf().setMaster("local[2]").setAppName("NetworkWordCount")
/***
* 创建StreamingContext需要sparkConf和batch interval
*/
val ssc=new StreamingContext(sparkConf,Seconds(5))
val lines=ssc.socketTextStream("bigdata.ibeifeng.com",6789)
val result= lines.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)
result.print()
ssc.start()
ssc.awaitTermination()
}
}
3.测试
nc -lk 6789