SparkCore系列(三)广播变量和累加器

一：广播变量

//广播变量其实就是将一个变量传播到每个excetor,实现excetor共享同一个只读变量.

//其中有一个难题就是,动态广播变量.我在实验过程中只是实现了不同job的广播变量更改,对于有N分钟修改一次还没有试验出更好的方法

import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.SparkContext
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.sql.SparkSession
import org.apache.spark.util.CollectionAccumulator

def main(args: Array[String]): Unit = {
   val sparkSession = SparkSession.builder()//.master("local")
      .appName("WordCount").getOrCreate()
   val sc = new JavaSparkContext(sparkSession.sparkContext).sc

   var runStatus = true;
   var broadcast:Broadcast[Long] = sc.broadcast(System.currentTimeMillis())
   class ThreadExample(sc : SparkContext) extends Thread{
      override def run(): Unit ={
         while(runStatus){
            val time = System.currentTimeMillis()
            broadcast.unpersist()
            broadcast = sc.broadcast(time)
            Thread.sleep(1000)
         }
      }
   }
   new ThreadExample(sc).start()
   var firstrdd = sc.textFile("").map(x=>broadcast)
   firstrdd.saveAsTextFile("")
   var secondrdd = sc.textFile("").map(x=>broadcast)
   firstrdd.saveAsTextFile("")

runStatus = false
}

二：累加器

//累加器实际上就是共享变量,实现多个excetor对同一份变量的多次操作

  val sparkSession = SparkSession.builder()//.master("local")
      .appName("WordCount").getOrCreate()
  val sc = new JavaSparkContext(sparkSession.sparkContext).sc

// 内置的累加器有三种，LongAccumulator、DoubleAccumulator、CollectionAccumulator// 内置的累加器有三种，LongAccumulator、DoubleAccumulator、 CollectionAccumulator

  // LongAccumulator: 数值型累加
  val longAccumulator = sc.longAccumulator("long-account")
  // DoubleAccumulator: 小数型累加
  val doubleAccumulator = sc.doubleAccumulator("double-account")
  // CollectionAccumulator：集合累加
  val collectionAccumulator:CollectionAccumulator[Int] = sc.collectionAccumulator("collection-account")

  var firstrdd = sc.textFile("s3://transsion-bigdata-warehouse/ods/athena_10410001/dt=20191210/hour=00/*")
      .map(x=>{
          longAccumulator.value;
          doubleAccumulator.add(1)
          collectionAccumulator.add(1)
          x
      })
  firstrdd.saveAsTextFile("s3://transsion-athena/test/push_msg/first")

  print("longAccumulator:"+longAccumulator.count)
  print("doubleAccumulator:"+doubleAccumulator.count)
  print("collectionAccumulator:"+collectionAccumulator.value.size())
  //longAccumulator:0 doubleAccumulator:2817336 collectionAccumulator:2817336