目的：
通过SBT编译scala程序然后在Kubernetes使用Spark运行WordCount任务
1: 安装环境和编译
安装SBT
curl https://bintray.com/sbt/rpm/rpm > bintray-sbt-rpm.repo
mv bintray-sbt-rpm.repo /etc/yum.repos.d/
yum install -y sbt
编辑文件
mkdir -p spark-example-project/src/main/scala
cd spark-example-project
ls /opt/spark/spark-2.1.1-bin-hadoop2.7/jars/scala-library-2.11.8.jar
ls /opt/spark/spark-2.1.1-bin-hadoop2.7/jars/spark-core_2.11-2.1.1.jar
simple.sbt
name := "Simple Project"
version := "1.0"
scalaVersion := "2.11.8"
libraryDependencies += "org.apache.spark" % "spark-core_2.11" % "2.1.1"
src/main/scala/SparkPi.scala
package org.apache.spark.examples
import scala.math.random
import org.apache.spark._
/** Computes an approximation to pi */
object SparkPi {
def main(args: Array[String]) {
val conf = new SparkConf().setAppName("Spark Pi")
val spark = new SparkContext(conf)
val slices = if (args.length > 0) args(0).toInt else 2
val n = math.min(100000L * slices, Int.MaxValue).toInt // avoid overflow
val count = spark.parallelize(1 until n, slices).map { i =>
val x = random * 2 - 1
val y = random * 2 - 1
if (x*x + y*y < 1) 1 else 0
}.reduce(_ + _)
println("Pi is roughly " + 4.0 * count / n)
spark.stop()
}
}
编译：
sbt clean
sbt package
2:打包运行
构建docker image
cp spark-example-project/target/scala-2.11/simple-project_2.11-1.0.jar /root/spark-2.3.0-bin-hadoop2.7/examples/jars/sparkpi_2.11-1.0.jar
cd /root/spark-2.3.0-bin-hadoop2.7
docker build -t 192.168.56.10:5000/spark:2.3.0.4 -f kubernetes/dockerfiles/spark/Dockerfile .
docker push 192.168.56.10:5000/spark:2.3.0.4
运行task
/root/spark/bin/spark-submit \
--master k8s://https://192.168.56.10:6443 \
--deploy-mode cluster \
--name spark-pi \
--class org.apache.spark.examples.SparkPi \
--conf spark.executor.instances=3 \
--conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \
--conf spark.kubernetes.container.image=192.168.56.10:5000/spark:2.3.0.4 \
local:///opt/spark/examples/jars/sparkpi_2.11-1.0.jar
