Spark常用action算子操作 —— Scala版

• collect算子

package rddDemo.action

import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by asus on 2018/6/17.
  * transformation 算子collect
  * 将数据从各从节点fetch到driver端进行集中处理（慎用）
  */
object CollectDemo {
  def main(args : Array[String]):Unit = {
    val conf = new SparkConf()
    conf.setAppName("CollectDemo")
    conf.setMaster("local[2]")

    System.setProperty("hadoop.home.dir" , "E:\\hadoop-2.6.0")

    val sc = new SparkContext(conf)

    val numbers = Array(1 to 10 : _*)
    val numRdd = sc.parallelize(numbers)

    /**
      * collect 算子将RDD数据从各从节点fetch到driver，当RDD数据量大时，容易造成driver端内存溢出OOM
      * 如果在集群模式下，当driver与client不在同一主机时，不使用collect，直接使用foreach进行打印输出，driver看不到输出结果
      * 因为输出结果在各从节点中
      */
    numRdd.map(x => x * 2).collect().foreach(println)

    sc.stop()
  }

}

• reduce算子

package rddDemo.action

import org.apache.spark.{SparkConf, SparkContext}

/**
  * ReduceByKey 算子
  * action 算子
  * Created by asus on 2018/7/7.
  */
object ReduceByKeyDemo {

  def main(args : Array[String]) : Unit = {
    val conf = new SparkConf()
    conf.setAppName("ReduceByKeyDemo")
    conf.setMaster("local[2]")

    System.setProperty("hadoop.home.dir" , "E:\\hadoop-2.6.0")

    val sc = new SparkContext(conf)

    val scores = List(("lao wang " , 10) , ("xiao wang" , 50) , ("lao zhang" , 50) , ("xiao zhang" , 100) ,
                        ("lao wang " , 10) , ("xiao wang" , 50) , ("lao zhang" , 50) , ("xiao zhang" , 100))

    val scoresRDD = sc.parallelize(scores , 3)
    scoresRDD.foreach(println)

    val totalScoresRDD = scoresRDD.reduceByKey(_ + _)
    totalScoresRDD.foreach(println)

    sc.stop()
  }

}

• take算子

package rddDemo.action

import org.apache.spark.{SparkConf, SparkContext}

/**
  * take(n)  获取RDD中的前 n 个元素
  * Created by asus on 2018/7/8.
  */
object TakeDemo {
  def main(args : Array[String]) : Unit = {
    val conf = new SparkConf()
    conf.setAppName("TakeDemo")
    conf.setMaster("local[2]")

    System.setProperty("hadoop.home.dir" , "E:\\hadoop-2.6.0")

    val sc = new SparkContext(conf)

    val numbers = List(0 to 10 : _*)
    val numbersRDD = sc.parallelize(numbers , 3)

    val take3Num = numbersRDD.take(3)
    for(n <- take3Num.iterator) {
      println(n)
    }

    sc.stop()
  }

}

• takeSample算子

package rddDemo.action

import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by asus on 2018/7/8.
  */
object TakeSampleDemo {
  def main(args : Array[String]) : Unit = {
    val conf = new SparkConf()
    conf.setAppName("TakeSampleDemo")
    conf.setMaster("local[2]")

    System.setProperty("hadoop.home.dir" , "E:\\hadoop-2.6.0")

    val sc = new SparkContext(conf)

    val names = List("lao wang" , "xiao wang" , "lao zhang" , "xiao zhang" , "lao li" , "xiao li")

    val namesRDD = sc.parallelize(names , 3)

    // 无放回取样，取 3 个元素
    println(">>>>>>>>>>>>>>>>>> 无放回取样，取 3 个元素 <<<<<<<<<<<<<<<<<<")
    val takeSample_1 = namesRDD.takeSample(false , 3) ;
    for(name <- takeSample_1) {
      println(name)
    }

    // 有放回取样，取 3 个元素
    println(">>>>>>>>>>>>>>>>>> 有放回取样，取 3 个元素 <<<<<<<<<<<<<<<<<<")
    val takeSample_2 = namesRDD.takeSample(true , 3)
    for(name <- takeSample_2) {
      println(name)
    }

    // 无放回取样，取 3 个元素，设置seed种子
    println(">>>>>>>>>>>>>>>>>> 无放回取样，取 3 个元素，设置seed种子 <<<<<<<<<<<<<<<<<<")
    val takeSample_3 = namesRDD.takeSample(false , 3 , 100)
    for(name <- takeSample_3) {
      println(name)
    }

    // 有放回取样，取 3 个元素，设置seed种子
    println(">>>>>>>>>>>>>>>>>> 有放回取样，取 3 个元素，设置seed种子 <<<<<<<<<<<<<<<<<<")
    val takeSample_4 = namesRDD.takeSample(true , 3 , 100)
    for(name <- takeSample_4) {
      println(name)
    }

    sc.stop()
  }

}

reduceByKey算子

package rddDemo.action

import org.apache.spark.{SparkConf, SparkContext}

/**
  * ReduceByKey 算子
  * action 算子
  * Created by asus on 2018/7/7.
  */
object ReduceByKeyDemo {

  def main(args : Array[String]) : Unit = {
    val conf = new SparkConf()
    conf.setAppName("ReduceByKeyDemo")
    conf.setMaster("local[2]")

    System.setProperty("hadoop.home.dir" , "E:\\hadoop-2.6.0")

    val sc = new SparkContext(conf)

    val scores = List(("lao wang " , 10) , ("xiao wang" , 50) , ("lao zhang" , 50) , ("xiao zhang" , 100) ,
                        ("lao wang " , 10) , ("xiao wang" , 50) , ("lao zhang" , 50) , ("xiao zhang" , 100))

    val scoresRDD = sc.parallelize(scores , 3)
    scoresRDD.foreach(println)

    val totalScoresRDD = scoresRDD.reduceByKey(_ + _)
    totalScoresRDD.foreach(println)

    sc.stop()
  }

}

• countByKey算子

package rddDemo.action

import org.apache.spark.{SparkConf, SparkContext}

/**
  * 计算 RDD key 出现的次数
  * Created by asus on 2018/7/15.
  */
object CountByKeyDemo {
  def main(args : Array[String]) : Unit = {
    val conf = new SparkConf()
    conf.setAppName("CountByKeyDemo") ;
    conf.setMaster("local[2]") ;

    System.setProperty("hadoop.home.dir" , "E:\\hadoop-2.6.0")

    val sc = new SparkContext(conf)

    val scores = List(("lao wang" , 10) , ("lao wang" , 20) , ("lao wang" , 30) ,
                      ("lao zhang" , 40) , ("xiao zhang" , 50) , ("xiao zhang" , 60) ,
                      ("xiao wang" , 70))

    val scoresRDD = sc.parallelize(scores , 2)
    val keyCount = scoresRDD.countByKey()
    for( s <- keyCount.keySet.iterator) {
      println("key -> " + s + " , " + "count -> " + keyCount.get(s))
    }

    sc.stop()
  }

}