Spark操作JDBC-API实现将数据存入到MySQL并读取出来(在小海豚里面操作)
代码:
bject mysql {
def main(args: Array[String]): Unit = {
//Spark支持通过Java JDBC访问关系型数据库。需要使用JdbcRDD
//创建
val conf = new SparkConf().setAppName("mysql").setMaster("local[*]")
val sc = new SparkContext(conf)
//插入数据
val data: RDD[(String, Int)] = sc.parallelize(List(("aaa",18),("bbb",19),("ccc",20)))
//调用foreachPartition针对每一个分区进行操作
data.foreachPartition(saveToMySQL)
//读取数据
def getConn():Connection={
DriverManager.getConnection("jdbc:mysql://localhost:3306/test?characterEncoding=UTF-8", "root", "root")
}
val studentRDD: JdbcRDD[(Int, String, Int)] = new JdbcRDD(sc,
getConn,
"select * from student where id >= ? and id <= ? ",
4,
6,
2,
rs => {
val id: Int = rs.getInt("id")
val name: String = rs.getString("name")
val age: Int = rs.getInt("age")
(id, name, age)
}
)
println(studentRDD.collect().toBuffer)
}
def saveToMySQL(partitionDate:Iterator[(String,Int)]):Unit={
//将数据存入到MySQL
//获取数据连接
val conn:Connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/test?characterEncoding=UTF-8", "root", "root")
partitionDate.foreach(data=>{
//将每一条数据存入到MySQL
val sql = "INSERT INTO `student` (`id`, `name`, `age`) VALUES (NULL, ?, ?);"
val ps: PreparedStatement = conn.prepareStatement(sql)
ps.setString(1,data._1)
ps.setInt(2,data._2)
ps.execute()
})
conn.close()
}
}
结果: