实验：MySQL使用JDBC执行批处理性能测试

试验准备

数据库：MySQL Server5.5.40

系统:Windows 7 ，Core(TM) i3-4160 @3.6HZ, Arch x64

数据库驱动：mysql-connector-java.5.1.9

JDK环境 JDK1.7.0_75 x64

实验要求

①每次插入10000条数据，然后删除，总共执行10次

②使用MyISAM和InnoDB引擎分别测试

③设置mysql数据包最大大小，防止缓存泄露max_allowed_packet = 8M

实验参考

测试mysql的sql语句预编译效果

MySQL批量插入性能优化(一)

MySQL批量插入性能优化（二）

扫描二维码关注公众号，回复： 20620 查看本文章

MySQL批量插入性能优化(三)

Statement和PreparedStatement批量更新

测试代码

建表代码

CREATE TABLE `user` (
  `id` int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT '用户主键',
  `name` varchar(32) NOT NULL COMMENT '用户名',
  `password` varchar(128) NOT NULL COMMENT '密码',
  `gender` varchar(8) NOT NULL COMMENT '性别',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=10001 DEFAULT CHARSET=utf8;

单元测试代码

package com.thin.mysql;

import java.sql.Connection;
import java.sql.Driver;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Enumeration;
import java.util.Random;

public class StatementTestCase {

	public static StatementTestCase instance;
	public static String url = "jdbc:mysql://localhost:3306/test?characterEncoding=utf8&useSSL=true&useServerPrepStmts=true";
	public Connection conn;
	public static String DriverClass = "com.mysql.jdbc.Driver";
	int MODE = 1000;

	public static StatementTestCase shareInstace() {
		if (instance == null) {
			synchronized (StatementTestCase.class) {
				if (instance == null) {
					instance = new StatementTestCase();
				}
			}
		}
		instance.openConnection();
		return instance;
	}

	private Driver findDriver(String driverName) {
		Enumeration<Driver> drivers = DriverManager.getDrivers();
		if (drivers != null) {
			do {
				if (!drivers.hasMoreElements()) {
					break;
				}
				Driver driver = drivers.nextElement();
				if (driver.getClass().getName().equals(driverName)) {
					return driver;
				}
			} while (true);
		}
		return null;
	}

	private synchronized void openConnection() {
		try {
			Driver driver = findDriver(DriverClass);
			if (driver == null) {
				Class.forName(DriverClass);
			}
			if (conn == null || conn.isClosed()) {
				conn = DriverManager.getConnection(url, "root", "root");
			}
		} catch (SQLException e) {
			e.printStackTrace();
		} catch (ClassNotFoundException e) {
			e.printStackTrace();
		}
	}

	public void executeStatement01() throws SQLException {
		long startTime = System.currentTimeMillis();
		Statement stmt = conn.createStatement();
		conn.setAutoCommit(false);
		String sql = null;
		String name = null;
		String password = null;
		String gender = null;
		String[] genders = new String[] { "男", "女" };
		Random rnd = new Random(System.currentTimeMillis());
		String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
		for (int i = 1; i <= 10000; i++) {
			int intRand = rnd.nextInt();
			name = "'" + "zhangsan" + i + "'";
			password = "'" + "pass" + intRand + "'";
			intRand = Math.abs((intRand) % 2);
			gender = "'" + genders[intRand] + "'";
			String nextVal = "(" + i + "," + name + "," + password + ","
					+ gender + ")";
			sql = statSql + nextVal + ";";
			stmt.execute(sql);

		}
		stmt.execute("delete from user where 1=1;");
		conn.commit();
		stmt.close();
		System.out.println("final : "
				+ (System.currentTimeMillis() - startTime));

	}

	public void executeStatement02() throws SQLException {
		long startTime = System.currentTimeMillis();
		Statement stmt = conn.createStatement();
		conn.setAutoCommit(false);
		String sql = null;
		String name = null;
		String password = null;
		String gender = null;
		String[] genders = new String[] { "男", "女" };
		Random rnd = new Random(System.currentTimeMillis());
		String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
		for (int i = 1; i <= 10000; i++) {
			int intRand = rnd.nextInt();
			name = "'" + "zhangsan" + i + "'";
			password = "'" + "pass" + intRand + "'";
			intRand = Math.abs((intRand) % 2);
			gender = "'" + genders[intRand] + "'";
			String nextVal = "(" + i + "," + name + "," + password + ","
					+ gender + ")";
			sql = statSql + nextVal + ";";
			stmt.addBatch(sql);

		}
		stmt.addBatch("delete from user where 1=1;");
		stmt.executeBatch();
		conn.commit();
		stmt.close();
		System.out.println("final : "
				+ (System.currentTimeMillis() - startTime));

	}

	public void executeStatement03() throws SQLException {
		long startTime = System.currentTimeMillis();
		Statement stmt = conn.createStatement();
		conn.setAutoCommit(false);
		String sql = null;
		String name = null;
		String password = null;
		String gender = null;
		String[] genders = new String[] { "男", "女" };
		Random rnd = new Random(System.currentTimeMillis());
		String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
		sql = statSql;
		for (int i = 1; i <= 10000; i++) {
			int intRand = rnd.nextInt();
			name = "'" + "zhangsan" + i + "'";
			password = "'" + "pass" + intRand + "'";
			intRand = Math.abs((intRand) % 2);
			gender = "'" + genders[intRand] + "'";
			String nextVal = "(" + i + "," + name + "," + password + ","
					+ gender + ")";
			sql = sql + nextVal + ",";

		}
		if (!sql.equalsIgnoreCase(statSql) && sql.endsWith(",")) {
			sql = sql.substring(0, sql.length() - 1);
			stmt.addBatch(sql);
		}
		stmt.addBatch("delete from user where 1=1;");
		stmt.executeBatch();
		conn.commit();
		stmt.close();
		System.out.println(sql);
		System.out.println("final : "
				+ (System.currentTimeMillis() - startTime));

	}

	public void executeStatement04() throws SQLException {
		long startTime = System.currentTimeMillis();
		Statement stmt = conn.createStatement();
		conn.setAutoCommit(false);
		String sql = null;
		String name = null;
		String password = null;
		String gender = null;
		String[] genders = new String[] { "男", "女" };
		Random rnd = new Random(System.currentTimeMillis());
		String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
		sql = statSql;
		for (int i = 1; i <= 10000; i++) {
			int intRand = rnd.nextInt();
			name = "'" + "zhangsan" + i + "'";
			password = "'" + "pass" + intRand + "'";
			intRand = Math.abs((intRand) % 2);
			gender = "'" + genders[intRand] + "'";
			String nextVal = "(" + i + "," + name + "," + password + ","
					+ gender + ")";
			sql = sql + nextVal + ",";
			if (i % 500 == 0) {
				if (!sql.equalsIgnoreCase(statSql) && sql.endsWith(",")) {
					sql = sql.substring(0, sql.length() - 1);
					stmt.execute(sql);
				}
				sql = statSql;
			}

		}
		if (!sql.equalsIgnoreCase(statSql) && sql.endsWith(",")) {
			sql = sql.substring(0, sql.length() - 1);
			stmt.execute(sql);
		}
		stmt.execute("delete from user where 1=1;");
		conn.commit();
		stmt.close();
		System.out.println("final : "
				+ (System.currentTimeMillis() - startTime));

	}

	public void executePrepareStatement01() throws SQLException {
		long startTime = System.currentTimeMillis();
		String sql = null;
		String name = null;
		String password = null;
		String gender = null;

		String[] genders = new String[] { "男", "女" };
		sql = "REPLACE INTO user (id,name,password,gender) VALUE (?,?,?,?)";
		PreparedStatement pstat = conn.prepareStatement(sql);
		conn.setAutoCommit(false);
		Random rnd = new Random(System.currentTimeMillis());
		for (int i = 1; i <= 10000; i++) {
			int intRand = Math.abs(rnd.nextInt() % 2);
			name = "zhangsan" + i;
			password = "pass" + intRand;
			intRand = Math.abs((intRand) % 2);

			gender = genders[intRand];

			pstat.clearParameters();
			pstat.setInt(1, i);
			pstat.setString(2, name);
			pstat.setString(3, password);
			pstat.setString(4, gender);
			pstat.execute();

		}
		pstat.execute("delete from user where 1=1;");
		conn.commit();
		pstat.close();

		System.out.println("final : "
				+ (System.currentTimeMillis() - startTime));
	}

	public void executePrepareStatement02() throws SQLException {
		long startTime = System.currentTimeMillis();
		String sql = null;
		String name = null;
		String password = null;
		String gender = null;

		String[] genders = new String[] { "男", "女" };
		sql = "REPLACE INTO user (id,name,password,gender) VALUE (?,?,?,?)";
		PreparedStatement pstat = conn.prepareStatement(sql);
		conn.setAutoCommit(false);
		Random rnd = new Random(System.currentTimeMillis());
		for (int i = 1; i <= 10000; i++) {
			int intRand = Math.abs(rnd.nextInt() % 2);
			name = "zhangsan" + i;
			password = "pass" + intRand;
			intRand = Math.abs((intRand) % 2);

			gender = genders[intRand];

			pstat.setInt(1, i);
			pstat.setString(2, name);
			pstat.setString(3, password);
			pstat.setString(4, gender);
			pstat.addBatch();

		}
		pstat.addBatch("delete from user where 1=1;");
		pstat.executeBatch();
		conn.commit();
		pstat.close();

		System.out.println("final : "
				+ (System.currentTimeMillis() - startTime));
	}

	public void executeStatement04_2() throws SQLException {
		long startTime = System.currentTimeMillis();
		Statement stmt = conn.createStatement();
		conn.setAutoCommit(false);
		String sql = null;
		String name = null;
		String password = null;
		String gender = null;
		String[] genders = new String[] { "男", "女" };
		Random rnd = new Random(System.currentTimeMillis());
		String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
		sql = statSql;
		for (int i = 1; i <= 10000; i++) {
			int intRand = rnd.nextInt();
			name = "'" + "zhangsan" + i + "'";
			password = "'" + "pass" + intRand + "'";
			intRand = Math.abs((intRand) % 2);
			gender = "'" + genders[intRand] + "'";
			String nextVal = "(" + i + "," + name + "," + password + ","
					+ gender + ")";
			sql = sql + nextVal + ",";
			if (i % MODE == 0) {
				if (!sql.equalsIgnoreCase(statSql) && sql.endsWith(",")) {
					sql = sql.substring(0, sql.length() - 1);
					stmt.execute(sql);
				}
				sql = statSql;
			}
		}
		if (!sql.equalsIgnoreCase(statSql) && sql.endsWith(",")) {
			sql = sql.substring(0, sql.length() - 1);
			stmt.execute(sql);
		}
		stmt.execute("delete from user where 1=1;");
		conn.commit();
		stmt.close();
		System.out.println("final : "
				+ (System.currentTimeMillis() - startTime));

	}

	public void executeStatement02_2() throws SQLException {
		long startTime = System.currentTimeMillis();
		Statement stmt = conn.createStatement();
		conn.setAutoCommit(false);
		String sql = null;
		String name = null;
		String password = null;
		String gender = null;
		String[] genders = new String[] { "男", "女" };
		Random rnd = new Random(System.currentTimeMillis());
		String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
		for (int i = 1; i <= 10000; i++) {
			int intRand = rnd.nextInt();
			name = "'" + "zhangsan" + i + "'";
			password = "'" + "pass" + intRand + "'";
			intRand = Math.abs((intRand) % 2);
			gender = "'" + genders[intRand] + "'";
			String nextVal = "(" + i + "," + name + "," + password + ","
					+ gender + ")";
			sql = statSql + nextVal + ";";
			stmt.addBatch(sql);
			if (i % MODE == 0) {
				stmt.executeBatch();
			}

		}
		stmt.addBatch("delete from user where 1=1;");
		stmt.executeBatch();
		conn.commit();
		stmt.close();
		System.out.println("final : "
				+ (System.currentTimeMillis() - startTime));

	}

	public void executePrepareStatement02_02() throws SQLException {
		long startTime = System.currentTimeMillis();
		String sql = null;
		String name = null;
		String password = null;
		String gender = null;

		String[] genders = new String[] { "男", "女" };
		sql = "REPLACE INTO user (id,name,password,gender) VALUE (?,?,?,?)";
		PreparedStatement pstat = conn.prepareStatement(sql);
		conn.setAutoCommit(false);
		Random rnd = new Random(System.currentTimeMillis());
		for (int i = 1; i <= 10000; i++) {
			int intRand = Math.abs(rnd.nextInt() % 2);
			name = "zhangsan" + i;
			password = "pass" + intRand;
			intRand = Math.abs((intRand) % 2);

			gender = genders[intRand];

			pstat.setInt(1, i);
			pstat.setString(2, name);
			pstat.setString(3, password);
			pstat.setString(4, gender);
			pstat.addBatch();
			if (i % MODE == 0) {
				pstat.executeBatch();
			}

		}
		pstat.addBatch("delete from user where 1=1;");
		pstat.executeBatch();
		conn.commit();
		pstat.close();

		System.out.println("final : "
				+ (System.currentTimeMillis() - startTime));
	}

}

测试调用代码实例

 public static void main( String[] args )
    {
       
    	try {
    		System.out.println("executeStatement04_2");
    		for (int i = 0; i < 10; i++) {
    			StatementTestCase.shareInstace().executeStatement04_2();
            }
    		
		} catch (SQLException e) {
			e.printStackTrace();
		}
    	
    }

测试结果

executeStatement01
final : 1616
final : 1124
final : 995
final : 1213
final : 1055
final : 1245
final : 1339
final : 1055
final : 1506
final : 1305
executeStatement02
final : 1504
final : 1282
final : 1416
final : 1136
final : 1186
final : 1171
final : 1130
final : 1776
final : 1369
final : 1105
executeStatement03
final : 4700
final : 5141
final : 4223
final : 4308
final : 4200
final : 4257
final : 4459
final : 4935
final : 4198
final : 4274
executeStatement04
final : 4271
final : 4310
final : 4167
final : 4581
final : 4159
final : 4259
final : 4657
final : 4384
final : 4166
final : 4217
executePrepareStatement01
final : 1161
final : 1522
final : 1067
final : 998
final : 982
final : 946
final : 1005
final : 886
final : 1306
final : 896
executePrepareStatement02
final : 1149
final : 1024
final : 1701
final : 1137
final : 892
final : 873
final : 964
final : 1006
final : 1188
final : 948

结果总结

①以上我们使用了InnoDB支持事务的引擎，如果不使用事务executeStatement03和executeStatement04案例的速度最快，但是也是在4200毫秒和4200毫秒左右，不加事务测试太浪费时间，所以我们省去了没有事务的结果。

②我们根据【试验参考】可以得出，executeStatement03和executeStatement04理论上应该是最快的，但是在使用了jdbc驱动之后效果并不理想，我们有理由怀疑jdbc实现的方式。

在数据库中，我们通过执行executeStatement04拼接的SQL语句，效率在155毫秒，其他的语句反而更慢。

③使用了事务可以提高效率

④使用了预处理prepareStatement方式可以较明显的提高效率

⑤jdbc的addBatch&executeBatch执行效率并不稳定，但是总体高于未使用的（并不明显）

⑥预处理语句的第一次执行时很慢的。

改进试验

由于实验结论②得出的结论不符合我们的预期，此外，根据其他结论，批处理效率可以适当提高，我们根据三种不同条件下批处理性能较好的方法进行改造。

executeStatement02

executeStatement04

executePrepareStatement02

经查阅资料我们发现，

①适当调节批量的粒度也可以提高性能

② 数据库连接加上参数rewriteBatchedStatements=true可以提高批处理性能

<1>通过调节粒度来实现性能提高

1.我们首先设置max_allowed_packet = 8M

set global max_allowed_packet = 8*1024*1024

2.改进的代码，用MODE控制批次粒度

public void executeStatement04_2() throws SQLException
	{
		 long startTime = System.currentTimeMillis();
		 Statement stmt = conn.createStatement();
		 conn.setAutoCommit(false);
		 String sql = null;
		 String name = null;
		 String password = null;
		 String gender = null;
		 String[] genders = new String[]{"男","女"};
		 Random rnd = new Random(System.currentTimeMillis());
		 String statSql="REPLACE INTO user(id,name,password,gender) VALUES ";
		 sql = statSql;
		 for (int i=1;i<=10000;i++)
		 {
			 int intRand = rnd.nextInt();
			 name = "'"+"zhangsan"+i+"'";
			 password = "'"+"pass"+intRand+"'";
			 intRand = Math.abs((intRand)%2);
			 gender = "'"+genders[intRand]+"'";
			 String nextVal =  "("+i+","+name+","+password+","+gender+")";
			 sql =  sql +  nextVal+",";
			 if(i%MODE==0)
			 {
				 if(!sql.equalsIgnoreCase(statSql) && sql.endsWith(","))
				 {
					 sql = sql.substring(0,sql.length()-1); 
					 stmt.execute(sql);
				 }
				 sql = statSql;
			 }
		 }
		 if(!sql.equalsIgnoreCase(statSql) && sql.endsWith(","))
		 {
			 sql = sql.substring(0,sql.length()-1); 
			 stmt.execute(sql);
		 }
		 stmt.execute("delete from user where 1=1;");
		 conn.commit();
		 stmt.close();
		 System.out.println("final : "+(System.currentTimeMillis()-startTime));
		
	}
	
	public void executeStatement02_2() throws SQLException  
	{
		 long startTime = System.currentTimeMillis();
		 Statement stmt = conn.createStatement();
		 conn.setAutoCommit(false);
		 String sql = null;
		 String name = null;
		 String password = null;
		 String gender = null;
		 String[] genders = new String[]{"男","女"};
		 Random rnd = new Random(System.currentTimeMillis());
		 String statSql="REPLACE INTO user(id,name,password,gender) VALUES ";
		 for (int i=1;i<=10000;i++)
		 {
			 int intRand = rnd.nextInt();
			 name = "'"+"zhangsan"+i+"'";
			 password = "'"+"pass"+intRand+"'";
			 intRand = Math.abs((intRand)%2);
			 gender = "'"+genders[intRand]+"'";
			 String nextVal =  "("+i+","+name+","+password+","+gender+")";
			 sql =  statSql +  nextVal +";";
			 stmt.addBatch(sql);
			 if(i%MODE==0)
			 {
				 stmt.executeBatch();
			 }
			
		 }
		 stmt.addBatch("delete from user where 1=1;");
		 stmt.executeBatch();
		 conn.commit();
		 stmt.close();
		 System.out.println("final : "+(System.currentTimeMillis()-startTime));
		
	}
	
	public void executePrepareStatement02_02() throws SQLException
	{
		 long startTime = System.currentTimeMillis();
		 String sql = null;
		 String name = null;
		 String password = null;
		 String gender = null;
		 
		 String[] genders = new String[]{"男","女"};
		 sql="REPLACE INTO user (id,name,password,gender) VALUE (?,?,?,?)";
		 PreparedStatement pstat = conn.prepareStatement(sql);
		 conn.setAutoCommit(false);
		 Random rnd = new Random(System.currentTimeMillis());
		 for (int i=1;i<=10000;i++)
		 {
			 int intRand = Math.abs(rnd.nextInt()%2);
			 name = "zhangsan"+i;
			 password = "pass"+intRand;
			 intRand = Math.abs((intRand)%2);
			 
			 gender = genders[intRand];
			 
			 pstat.setInt(1, i);
			 pstat.setString(2, name);
			 pstat.setString(3, password);
			 pstat.setString(4, gender);
			 pstat.addBatch();
			 if(i%MODE==0)
			 {
				 pstat.executeBatch();
			 }
			
		 }
		 pstat.addBatch("delete from user where 1=1;");
		 pstat.executeBatch();
		 conn.commit();
		 pstat.close();
		 
		 System.out.println("final : "+(System.currentTimeMillis()-startTime));
	}

测试结果

MODE=500

executeStatement02_2
final : 1558
final : 1228
final : 1070
final : 1477
final : 1002
final : 1024
final : 1003
final : 1129
final : 1415
final : 1306
executeStatement04_2
final : 464
final : 511
final : 538
final : 523
final : 568
final : 396
final : 555
final : 429
final : 672
final : 449
executePrepareStatement02_02
final : 1434
final : 968
final : 991
final : 975
final : 1015
final : 1099
final : 1180
final : 1001
final : 889
final : 1291

MODE=1000

executeStatement02_2
final : 1860
final : 1420
final : 1502
final : 1195
final : 1395
final : 1135
final : 1139
final : 1652
final : 1200
final : 1433
executeStatement04_2
final : 642
final : 888
final : 562
final : 603
final : 853
final : 556
final : 595
final : 820
final : 637
final : 596
executePrepareStatement02_02
final : 1663
final : 1281
final : 1131
final : 1381
final : 1341
final : 1230
final : 1189
final : 1227
final : 1585
final : 1231

以上可以得出MODE为500性能比较出众

<2>修改连接参数提高性能

public static String url =”jdbc:mysql://localhost:3306/test?characterEncoding=utf8&useSSL=true&useServerPrepStmts=true&rewriteBatchedStatements=true“

我们以mode=500，开启事务下进行测试，结果如下

executeStatement02_2
java.sql.BatchUpdateException: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ';REPLACE INTO user(id,name,password,gender) VALUES (2,'zhangsan2','pass-18490631' at line 1
	at com.mysql.jdbc.StatementImpl.executeBatchUsingMultiQueries(StatementImpl.java:1231)
	at com.mysql.jdbc.StatementImpl.executeBatch(StatementImpl.java:1012)
	at com.yyuap.tianwt.StatementTestCase.executeStatement02_2(StatementTestCase.java:350)
	at com.yyuap.tianwt.App.main(App.java:16)
	
executeStatement04_2
final : 448
final : 487
final : 718
final : 388
final : 505
final : 546
final : 553
final : 796
final : 522
final : 602
	
executePrepareStatement02_02
final : 997
final : 897
final : 899
final : 932
final : 880
final : 816
final : 882
final : 846
final : 745
final : 758

改造后的结论

①通过以上条件MODE方式控制粒度，批量提交，addBatch&executeBatch效率并没有明显提高

②MODE在500时表现最为突出

③.MODE+修改连接参数的情况下，预编译语句的批处理明显提高

④非预编译不支持rewriteBatchedStatements=true的addBatch&executeBatch

最终结论

通过以上实验和改造后的实验，最终结论：

①多Values（如executeStatement04_2案例）拼接在非事务状态下效率比一般的SQL要高，可以让效率提高数十倍多

②使用事务可以让效率提高数十倍多

③事务+多Values在粒度500-1000中执行批处理效率最高

④addBatch&executeBatch在预编译prepareStatement条件下，设置rewriteBatchedStatements=true可以明显提高批处理性能

⑤预编译SQL总体效率高，但是第一次执行效率很低