import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; /** * 适用于大数据量的导入操作 * @author qth * */ public abstract class SingleThreadSaver { private Map<String, Long> countHolder = null; private List<String> errorMsgHolder = null; private static final String COUNT_KEY_SUCCESS = "success"; private static final String COUNT_KEY_ERROR = "error"; private SingleThreadSaverPool pool = null; public SingleThreadSaver() { countHolder = new HashMap<String, Long>(); countHolder.put(COUNT_KEY_SUCCESS, 0L); countHolder.put(COUNT_KEY_ERROR, 0L); errorMsgHolder = new ArrayList<String>(); pool = SingleThreadSaverPool.getInstance(); } public List<String> getErrorMsgList() { List<String> ret = new ArrayList<String>(); ret.addAll(errorMsgHolder); return ret; } protected synchronized void increaseSuccessCount() { countHolder.put(COUNT_KEY_SUCCESS, countHolder.get(COUNT_KEY_SUCCESS) + 1); } public long getSuccessCount() { return countHolder.get(COUNT_KEY_SUCCESS); } protected synchronized void increaseErrorCount() { countHolder.put(COUNT_KEY_ERROR, countHolder.get(COUNT_KEY_ERROR) + 1); } public long getErrorCount() { return countHolder.get(COUNT_KEY_ERROR); } protected void addErrorMsg(String msg) { errorMsgHolder.add(msg); } public void saveInSingleThread() { final Date now = new Date(); new Thread(new Runnable() { @Override public void run() { try { save(); } catch(Exception e) { e.printStackTrace(); addErrorMsg(e.getMessage()); increaseErrorCount(); } finally { try { pool.releaseSaver(SingleThreadSaver.this); } catch(Exception e) { e.printStackTrace(); } } } }).start(); } /** * 单线程执行体 */ protected abstract void save(); }
===========================================
package com.nssc.exam.common; import java.util.HashMap; import java.util.Map; /** * 利用线程池管理SingleThreadSaver, 每种Class的SingleThreadSaver同一时间中只允许运行5个 * @author qth * */ public final class SingleThreadSaverPool { private static SingleThreadSaverPool instance = null; private static Map<Class<? extends SingleThreadSaver>, Map<SingleThreadSaver, Integer>> pool = null; private static final int POOL_SIZE_DEFAULT = 5; private static final Integer STATE_FREE = 0; private static final Integer STATE_BUSY = 1; private SingleThreadSaverPool() { pool = new HashMap<Class<? extends SingleThreadSaver>, Map<SingleThreadSaver,Integer>>(); } public static SingleThreadSaverPool getInstance() { if(instance == null) { instance = new SingleThreadSaverPool(); } return instance; } /** * 取得一个空闲的 saver * @param clazz * @return * @throws Exception */ public static <T extends SingleThreadSaver> T getSaver( Class<? extends SingleThreadSaver> clazz) throws Exception { if(instance == null) { instance = new SingleThreadSaverPool(); } Map<SingleThreadSaver, Integer> subPool = pool.get(clazz); if(subPool == null || subPool.isEmpty()) { subPool = new HashMap<SingleThreadSaver, Integer>(); for(int i = 0; i < POOL_SIZE_DEFAULT; i++) { SingleThreadSaver saverTmp = clazz.newInstance(); subPool.put(saverTmp, STATE_FREE); } pool.put(clazz, subPool); } for(Map.Entry<SingleThreadSaver, Integer> entry : subPool.entrySet()) { SingleThreadSaver tmp = entry.getKey(); if(subPool.get(tmp) == STATE_FREE) { subPool.put(tmp, STATE_BUSY); return (T)tmp; } } //now all 5 are busy throw new IllegalStateException("No free saver available, please try later."); } /** * 释放得到的saver * @param <T> * @param saver */ public static <T extends SingleThreadSaver> void releaseSaver(T saver) throws Exception { Map<SingleThreadSaver, Integer> subPool = pool.get(saver.getClass()); if(subPool == null || subPool.isEmpty()) { throw new IllegalStateException("The saver passed is not get from pool!"); } if(subPool.get(saver) == STATE_FREE) { throw new IllegalStateException("Pool is NOT working properly!"); } subPool.put(saver, STATE_FREE); } }
===========================================
项目实例:大量照片从文件系统导入数据库
===========================================
import java.io.File; import java.util.Date; import com.nssc.exam.common.SingleThreadSaver; import com.nssc.exam.common.Tools; import com.nssc.exam.self.model.TStudentPhoto; public class SelfStudentPhotoSaver extends SingleThreadSaver { private IStudentService studentService = null; private File folder = null; private Date optTime = null; private long userId = -1L; @Override public void save() { importPhotosUnderThisFolder(folder, optTime, userId); } private void importPhotosUnderThisFolder(File pFolder, Date optTime, long userId) { if(pFolder == null || !pFolder.canRead()) { return; } if(pFolder.isFile()) { try { if(!pFolder.getName().toUpperCase().endsWith(".JPG")) { return; } String fileName = pFolder.getName().toUpperCase().trim(); String zkz = fileName.substring(0, fileName.indexOf(".JPG")); if(zkz.length() > 12) { addErrorMsg("准考证不合法:" + zkz + "(" + pFolder.getAbsolutePath() + ")"); increaseErrorCount(); return; } TStudentPhoto sp = new TStudentPhoto(); sp.setZkz(zkz); sp.setPhotoContent(Tools.getBytesFromFile(pFolder)); sp.setOperateTime(optTime); sp.setOperateUserId(userId); if(studentService.uploadStudentPhoto(sp)) { increaseSuccessCount(); } else { increaseErrorCount(); } return; } catch(Exception e) { e.printStackTrace(); addErrorMsg(e.getMessage()); increaseErrorCount(); return; } } if(pFolder.listFiles() == null) { return; } for(File subFolder : pFolder.listFiles()) { importPhotosUnderThisFolder(subFolder, optTime, userId); } } public static void main(String[] args) throws Exception{ SelfStudentPhotoSaver saver = SelfStudentPhotoSaver.class.newInstance(); System.out.println(saver); } public IStudentService getStudentService() { return studentService; } public void setStudentService(IStudentService studentService) { this.studentService = studentService; } public File getFolder() { return folder; } public void setFolder(File folder) { this.folder = folder; } public Date getOptTime() { return optTime; } public void setOptTime(Date optTime) { this.optTime = optTime; } public long getUserId() { return userId; } public void setUserId(long userId) { this.userId = userId; } }