Action Directory
- reduce(f: (T, T) => T): T
- collect(): Array[T]
- count(): Long
- first(): T
- take(num: Int): Array[T]
- takeOrdered(num: Int)(implicit ord: Ordering[T])
- aggregate[U: ClassTag](zeroValue: U)(seqOp: (U, T) => U, combOp: (U, U) => U): U
- fold(zeroValue: T)(op: (T, T) => T): T
- saveAsTextFile(path: String): Unit
- saveAsObjectFile(path: String): Unit
- countByKey(): Map[K, Long]
- foreach(f: T => Unit): Unit
reduce(f: (T, T) => T): T
: Gather all the elements in the RDD by function func
scala> var rdd=sc.parallelize(1 to 10,2).collect
rdd: Array[Int] = Array(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
scala> rdd.reduce(_+_)
res1: Int = 55
scala> sc.makeRDD(Array(("A",0),("A",2),("B",1),("B",2),("C",1)))
res3: org.apache.spark.rdd.RDD[(String, Int)] = ParallelCollectionRDD[3] at makeRDD at <console>:25
scala> res3.reduce((x,y)=>(x._1+y._1,x._2+y._2))
res4: (String, Int) = (BBCAA,6)
collect(): Array[T]
: In the driver returns the data set all the elements of an array
scala> var rdd=sc.parallelize(1 to 10,2).collect
rdd: Array[Int] = Array(1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
count(): Long
: Returns the number of elements in the RDD
scala> var rdd=sc.makeRDD(1 to 10)
rdd: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[5] at makeRDD at <console>:24
scala> rdd.count
res9: Long = 10
first(): T
: Returns the first element in the RDD
scala> var rdd=sc.makeRDD(1 to 10)
rdd: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[6] at makeRDD at <console>:24
scala> rdd.first
res10: Int = 1
take(num: Int): Array[T]
: Returns the RDD first n elements
scala> var rdd=sc.makeRDD(1 to 10)
rdd: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[7] at makeRDD at <console>:24
scala> rdd.take(3)
res11: Array[Int] = Array(1, 2, 3)
takeOrdered(num: Int)(implicit ord: Ordering[T])
: Returns the first few ranking
scala> var rdd=sc.makeRDD(Array(1,3,6,2,4,5))
rdd: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[10] at makeRDD at <console>:24
scala> rdd.takeOrdered(3)
res14: Array[Int] = Array(1, 2, 3)
aggregate[U: ClassTag](zeroValue: U)(seqOp: (U, T) => U, combOp: (U, U) => U): U
: Aggregate function for each partition inside the element by polymerizing seqOp initial value, and the result of the initial value of each partition (zeroValue) and then combine with a combine operation function. This function returns the type is not required and the final element type consistent with the RDD.
scala> var rdd=sc.makeRDD(1 to 10,2)
rdd: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[12] at makeRDD at <console>:24
scala> rdd.aggregate(1)(
| (x:Int,y:Int)=>x+y,
| (a:Int,b:Int)=>a+b
| )
res16: Int = 58
scala> rdd.aggregate(1)(
| (x:Int,y:Int)=>x*y,
| (a:Int,b:Int)=>a*b
| )
res17: Int = 3628800
fold(zeroValue: T)(op: (T, T) => T): T
: Folding operation, to simplify the operation of the aggregate, as SEQOP and combop.
scala> var rdd=sc.makeRDD(1 to 10,2)
rdd: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[12] at makeRDD at <console>:24
scala> rdd.aggregate(1)(
| (x:Int,y:Int)=>x+y,
| (a:Int,b:Int)=>a+b
| )
res16: Int = 58
scala> rdd.fold(1)(_+_)
res18: Int = 58
saveAsTextFile(path: String): Unit
: The RDD as a text file saved locally or in HDFS
scala> var rdd=sc.makeRDD(1 to 10,2)
rdd: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[12] at makeRDD at <console>:24
scala> rdd.saveAsTextFile("hdfs://hadoop01:9000/rdd")
[root@hadoop01 ~]# hadoop fs -cat /rdd/*
1
2
3
4
5
6
7
8
9
10
saveAsObjectFile(path: String): Unit
: The RDD to the elements stored locally or HDFS to form the serialized objects.
scala> rdd.saveAsObjectFile("hdfs://hadoop01:9000/rdd")
countByKey(): Map[K, Long]
: For (K, V) type RDD, a return (K, Int) of the map, represents the number of elements corresponding to each key.
scala> var rdd=sc.makeRDD(List((1,3),(1,2),(1,4),(2,3),(3,6),(3,8)),3)
rdd: org.apache.spark.rdd.RDD[(Int, Int)] = ParallelCollectionRDD[17] at makeRDD at <console>:24
scala> rdd.countByKey()
res22: scala.collection.Map[Int,Long] = Map(3 -> 2, 1 -> 3, 2 -> 1)
foreach(f: T => Unit): Unit
: On each element of the data set, function func update operation.
scala> var rdd=sc.makeRDD(1 to 10 ,2)
rdd: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[20] at makeRDD at <console>:24
scala> rdd.collect().foreach(println)
1
2
3
4
5
6
7
8
9
10