spark--StructuredStreaming与其他技术整合-★★★★

Kafka

准备依赖

 <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-sql-kafka-0-10_2.11</artifactId>
        <version>${spark.version}</version>
    </dependency>

官网API介绍

http://spark.apache.org/docs/2.2.0/structured-streaming-kafka-integration.html

注意

代码实现

package cn.hanjiaxiaozhi.structedstream
​
import org.apache.spark.SparkContext
import org.apache.spark.sql.{
    
    DataFrame, Dataset, Row, SparkSession}
​
/**
 * Author hanjiaxiaozhi
 * Date 2020/7/27 10:22
 * Desc StructuredStreaming整合Kafka,从Kafka消费数据
 */
object StructuredStreaming_Kafka {
    
    
  def main(args: Array[String]): Unit = {
    
    
    //1.准备环境
    val spark: SparkSession = SparkSession.builder.appName("wc").master("local[*]").getOrCreate()
    val sc: SparkContext = spark.sparkContext
    sc.setLogLevel("WARN")
    import spark.implicits._
​
    //2.连接Kafka获取数据
    val df: DataFrame = spark.readStream
      .format("kafka")
      .option("kafka.bootstrap.servers", "node01:9092")//kafka集群地址
      .option("subscribe", "spark_kafka")//要订阅的主题
      .load()
    //df中就有了从Kafka消费的数据,有固定的Schema,我们要获取其中的value是binary类型,所以需要通过如下转换,转为String类型
    val ds: Dataset[String] = df.selectExpr("CAST(value AS STRING)") //将binary类型的value反序列化为String
      .as[String]//将df转为有泛型的ds
​
    //3.处理数据
    val result: Dataset[Row] = ds.flatMap(_.split(" ")).groupBy("value").count().orderBy($"count".desc)
​
    //4.输出结果--先输出到控制台
    result.writeStream
      .format("console")
      .outputMode("complete")
      .option("truncate",false)//表示列如果过长不会以...省略
      .start()
      .awaitTermination()
​
    //  /export/servers/kafka/bin/kafka-console-producer.sh --broker-list node01:9092 --topic spark_kafka
​
  }
}
​

MySQL

准备MySQL表

CREATE TABLE `t_word` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `word` varchar(255) NOT NULL,
  `count` bigint(11) DEFAULT NULL,
  PRIMARY KEY (`id`),
  UNIQUE KEY `word` (`word`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
​

官网和博客说明

• Spark母公司的博客上有上述代码的示例程序

https://databricks.com/blog/2017/04/04/real-time-end-to-end-integration-with-apache-kafka-in-apache-sparks-structured-streaming.html

• Databricks是由Apache Spark的创始人建立的商业公司
  

代码实现

package cn.hanjiaxiaozhi.structedstream
​
import java.sql.{
    
    Connection, DriverManager, PreparedStatement}
​
import org.apache.spark.SparkContext
import org.apache.spark.sql.{
    
    DataFrame, Dataset, ForeachWriter, Row, SparkSession}
​
/**
 * Author hanjiaxiaozhi
 * Date 2020/7/27 10:22
 * Desc StructuredStreaming整合其他技术,
 * 从Kafka消费数据并做WordCount,最后将结果输出到MySQL
 */
object StructuredStreaming_Kafka_MySQL{
    
    
  def main(args: Array[String]): Unit = {
    
    
    //1.准备环境
    val spark: SparkSession = SparkSession.builder.appName("wc").master("local[*]").getOrCreate()
    val sc: SparkContext = spark.sparkContext
    sc.setLogLevel("WARN")
    import spark.implicits._
​
    //2.连接Kafka获取数据
    val df: DataFrame = spark.readStream
      .format("kafka")
      .option("kafka.bootstrap.servers", "node01:9092")//kafka集群地址
      .option("subscribe", "spark_kafka")//要订阅的主题
      .load()
    //df中就有了从Kafka消费的数据,有固定的Schema,我们要获取其中的value是binary类型,所以需要通过如下转换,转为String类型
    val ds: Dataset[String] = df.selectExpr("CAST(value AS STRING)") //将binary类型的value反序列化为String
      .as[String]//将df转为有泛型的ds
​
    //3.处理数据
    val result: Dataset[Row] = ds.flatMap(_.split(" ")).groupBy("value").count().orderBy($"count".desc)
​
    //4.输出结果--输出到MySQL
    val jdbcSink = new JdbcSink("jdbc:mysql://localhost:3306/bigdata?characterEncoding=UTF-8", "root", "root")
    result.writeStream
      .foreach(jdbcSink)
      .outputMode("complete")
      .start()
      .awaitTermination()
​
  }
​
  class JdbcSink(url:String,username:String,password:String) extends ForeachWriter[Row]{
    
    
    var conn: Connection = null
    var ps: PreparedStatement = null
​
    //开启连接
    override def open(partitionId: Long, version: Long): Boolean = {
    
    
      conn = DriverManager. getConnection(url,username,password)
      val sql:String =
        """
          |REPLACE INTO `t_word` (`id`, `word`, `count`) VALUES (NULL, ?, ?);
          |""".stripMargin
      ps = conn.prepareStatement(sql)
      true //表示创建成功
    }
​
    //处理数据/保存数据到MySQL
    override def process(value: Row): Unit = {
    
    
      val word: String = value.getAs[String](0)
      val count: Long = value.getAs[Long](1)
      ps.setString(1,word)
      ps.setLong(2,count)
      ps.executeUpdate()
    }
​
    //关闭资源
    override def close(errorOrNull: Throwable): Unit = {
    
    
      if(conn != null) conn.close()
      if(ps != null) ps.close()
    }
  }
}
​