实验环境
windows10；
mysql8.0.15；
问题描述
在使用scala编写saprk程序连接数据库时出现如下所示报错：
com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException: Could not create connection to database server.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
at com.mysql.jdbc.Util.getInstance(Util.java:386)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1015)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:989)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:975)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:920)
at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2575)
at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2311)
at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:834)
at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:416)
at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:347)
at java.sql.DriverManager.getConnection(DriverManager.java:664)
at java.sql.DriverManager.getConnection(DriverManager.java:247)
at com.atguigu.bigdata.spark.Spark18_Mysql$$anonfun$1.apply(Spark18_Mysql.scala:34)
at com.atguigu.bigdata.spark.Spark18_Mysql$$anonfun$1.apply(Spark18_Mysql.scala:31)
at org.apache.spark.rdd.JdbcRDD$$anon$1.<init>(JdbcRDD.scala:82)
at org.apache.spark.rdd.JdbcRDD.compute(JdbcRDD.scala:78)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:402)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:408)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.NullPointerException
at com.mysql.jdbc.ConnectionImpl.getServerCharacterEncoding(ConnectionImpl.java:3286)
at com.mysql.jdbc.MysqlIO.sendConnectionAttributes(MysqlIO.java:1987)
at com.mysql.jdbc.MysqlIO.proceedHandshakeWithPluggableAuthentication(MysqlIO.java:1913)
at com.mysql.jdbc.MysqlIO.doHandshake(MysqlIO.java:1290)
at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2493)
at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2526)
... 26 more
19/11/20 14:18:14 ERROR Executor: Exception in task 0.0 in stage 0.0 (TID 0)
com.mysql.jdbc.exceptions.jdbc4.MySQLNonTransientConnectionException: Could not create connection to database server.
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
at com.mysql.jdbc.Util.getInstance(Util.java:386)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1015)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:989)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:975)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:920)
at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2575)
at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2311)
at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:834)
at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:416)
at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:347)
at java.sql.DriverManager.getConnection(DriverManager.java:664)
at java.sql.DriverManager.getConnection(DriverManager.java:247)
at com.atguigu.bigdata.spark.Spark18_Mysql$$anonfun$1.apply(Spark18_Mysql.scala:34)
at com.atguigu.bigdata.spark.Spark18_Mysql$$anonfun$1.apply(Spark18_Mysql.scala:31)
at org.apache.spark.rdd.JdbcRDD$$anon$1.<init>(JdbcRDD.scala:82)
at org.apache.spark.rdd.JdbcRDD.compute(JdbcRDD.scala:78)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:121)
at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:402)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:408)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.NullPointerException
at com.mysql.jdbc.ConnectionImpl.getServerCharacterEncoding(ConnectionImpl.java:3286)
at com.mysql.jdbc.MysqlIO.sendConnectionAttributes(MysqlIO.java:1987)
at com.mysql.jdbc.MysqlIO.proceedHandshakeWithPluggableAuthentication(MysqlIO.java:1913)
at com.mysql.jdbc.MysqlIO.doHandshake(MysqlIO.java:1290)
at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2493)
at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2526)
... 26 more
程序源码
package com.atguigu.bigdata.spark
import org.apache.spark.rdd.{JdbcRDD, RDD}
import org.apache.spark.{SparkConf, SparkContext}
import scala.util.parsing.json.JSON
/**
* Author: Solarzhou
* data: 2019/11/14 15:43
* desc:
*/
object Spark18_Mysql {
def main(args: Array[String]): Unit = {
val config: SparkConf = new SparkConf().setMaster("local[*]").setAppName("jdbcRDD")
// 创建上下文对象
val sc: SparkContext = new SparkContext(config)
val driver = "com.mysql.jdbc.Driver"
val url = "jdbc:mysql://localhost:3306/rdd"
// val url = "jdbc:mysql://localhost:3306/rdd"
val userName = "root"
val passWd = "000000"
// 创建jdbcRDD,访问数据库
val sql = "select name,age from user where ? <= id and id <= ?"
// val sql = "select task_id,timestamp from taskmanager_task_status where ? <= id and id <= ?"
val jdbcRDD = new JdbcRDD(
sc,
() => {
//获取数据库连接对象
Class.forName(driver)
java.sql.DriverManager.getConnection(url, userName, passWd)
},
sql,
1,
3,
2,
(rs) =>
(rs.getString(1), rs.getString(2))
)
println(jdbcRDD.count())
// jdbcRDD.collect()
jdbcRDD.foreach(println)
sc.stop()
}
}
- 配置文件pox.xml
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.27</version>
</dependency>
错误分析
笔者在stackouverflow上看到这是有版本不一致导致的。也就是我们使用maven下载的依赖与我们本地的mysql本部不兼容。
参考文章
解决方案
既然是版本不一致，那么我们查看本地的mysql版本，然后继续构造一直的版本就行了。
笔者使用的mysql8.0.15，重新加载依赖后错误解决；
报错二
上述报错解决后，重新运行程序，出现如下错误：
The server time zone value 'Öйú±ê׼ʱ¼ä' is unrecognized or represents more than one time zone
分析错误，容易知道是时间问题，我们需要在数据库 URL中设置serverTimezone属性，如：
val url = "jdbc:mysql://localhost:3306/rdd" + "?serverTimezone=GMT%2B8" 这里的 GMT%2B8 代表是东八区。
至此，所有报错消除，我们便可以使用scala编写spark程序来读写mysql数据库中的内容了。
从一名不羁的码农开始,谈风月之余谈技术
