java 文件下载 断点续传

最近闲来无事，因为公司屏蔽了迅雷软件的下载端口，所以自己写了一个下载工具。拿过来分享下。

下载网络上的文件肯定不能只用单线程下载，这样下载太慢，网速得不多合理利用。那么就应该用多线程下载和线程池调度线程。

所以我们要讲文件切分成N段下载。用到了RandomAccessFile 随机访问文件。

首先我们写一个主线程，用来管理下载的子线程：

package org.app.download.component;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.math.BigDecimal;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.Map;
import java.util.Set;

import org.app.download.core.EngineCore;

/**
 * 
 * @Title: DLTask.java
 * @Description: 本类对应一个下载任务，每个下载任务包含多个下载线程，默认最多包含十个下载线程
 * @Package org.app.download.component
 * @author [email protected]
 * @date 2012-8-1
 * @version V1.0
 * 
 */
public class DLTask extends Thread implements Serializable {

	private static final long serialVersionUID = 126148287461276024L;
	// 下载临时文件后缀，下载完成后将自动被删除
	public final static String FILE_POSTFIX = ".tmp";
	// URL地址
	private URL url;
	// 文件对象
	private File file;
	// 文件名
	private String filename;
	// 下载线程数量，用户可定制
	private int threadQut;
	// 下载文件长度
	private int contentLen;
	// 当前下载完成总数
	private long completedTot;
	// 下载时间计数，记录下载耗费的时间
	private int costTime;
	// 下载百分比
	private String curPercent;
	// 是否新建下载任务，可能是断点续传任务
	private boolean isNewTask;
	// 保存当前任务的线程
	private DLThread[] dlThreads;
	// 当前任务的监听器，用于即时获取相关下载信息
	transient private DLListener listener;

	public DLTask(int threadQut, String url, String filename) {
		this.threadQut = threadQut;
		this.filename = filename;
		costTime = 0;
		curPercent = "0";
		isNewTask = true;
		this.dlThreads = new DLThread[threadQut];
		this.listener = new DLListener(this);
		try {
			this.url = new URL(url);
		} catch (MalformedURLException ex) {
			ex.printStackTrace();
			throw new RuntimeException(ex);
		}
	}

	@Override
	public void run() {
		if (isNewTask) {
			newTask();
			return;
		}
		resumeTask();
	}

	/**
	 * 恢复任务时被调用，用于断点续传时恢复各个线程。
	 */
	private void resumeTask() {
		listener = new DLListener(this);
		file = new File(filename + FILE_POSTFIX);
		for (int i = 0; i < threadQut; i++) {
			dlThreads[i].setDlTask(this);
			EngineCore.pool.execute(dlThreads[i]);
		}
		EngineCore.pool.execute(listener);
	}

	/**
	 * 新建任务时被调用，通过连接资源获取资源相关信息，并根据具体长度创建线程块， 线程创建完毕后，即刻通过线程池进行调度
	 * 
	 * @throws RuntimeException
	 */
	@SuppressWarnings({ "rawtypes", "unchecked" })
	private void newTask() throws RuntimeException {
		try {
			isNewTask = false;
			URLConnection con = url.openConnection();
			Map map = con.getHeaderFields();
			Set<String> set = map.keySet();
			for (String key : set) {
				System.out.println(key + " : " + map.get(key));
			}
			contentLen = con.getContentLength();
			if (contentLen <= 0) {
				System.out.println("Unable to get resources length, the interrupt download process!");
				return;
			}
			file = new File(filename + FILE_POSTFIX);
			int fileCnt = 1;
			while (file.exists()) {
				file = new File(filename += (fileCnt + FILE_POSTFIX));
				fileCnt++;
			}

			int subLenMore = contentLen % threadQut;
			int subLen = (contentLen - subLenMore) / threadQut;

			for (int i = 0; i < threadQut; i++) {
				DLThread thread;
				if (i == threadQut - 1) {
					thread = new DLThread(this, i + 1, subLen * i, (subLen * (i + 1) - 1) + subLenMore);
				} else {
					thread = new DLThread(this, i + 1, subLen * i, subLen * (i + 1) - 1);
				}
				dlThreads[i] = thread;
				EngineCore.pool.execute(dlThreads[i]);
			}

			EngineCore.pool.execute(listener);
		} catch (IOException ex) {
			ex.printStackTrace();
			throw new RuntimeException(ex);
		}
	}

	/**
	 * 计算当前已经完成的长度并返回下载百分比的字符串表示，目前百分比均为整数
	 * 
	 * @return
	 */
	public String getCurPercent() {
		this.completeTot();
		curPercent = new BigDecimal(completedTot).divide(new BigDecimal(this.contentLen), 2, BigDecimal.ROUND_HALF_EVEN).divide(new BigDecimal(0.01), 0, BigDecimal.ROUND_HALF_EVEN).toString();
		return curPercent;
	}

	/**
	 * 获取当前下载的字节
	 */
	private void completeTot() {
		completedTot = 0;
		for (DLThread t : dlThreads) {
			completedTot += t.getReadByte();
		}
	}

	/**
	 * 判断全部线程是否已经下载完成，如果完成则返回true，相反则返回false
	 * 
	 * @return
	 */
	public boolean isComplete() {
		boolean completed = true;
		for (DLThread t : dlThreads) {
			completed = t.isFinished();
			if (!completed) {
				break;
			}
		}
		return completed;
	}

	/**
	 * 下载完成后重命名文件
	 */
	public void rename() {
		this.file.renameTo(new File(filename));
	}

	public DLThread[] getDlThreads() {
		return dlThreads;
	}

	public void setDlThreads(DLThread[] dlThreads) {
		this.dlThreads = dlThreads;
	}

	public File getFile() {
		return file;
	}

	public URL getUrl() {
		return url;
	}

	public int getContentLen() {
		return contentLen;
	}

	public String getFilename() {
		return filename;
	}

	public int getThreadQut() {
		return threadQut;
	}

	public long getCompletedTot() {
		return completedTot;
	}

	public int getCostTime() {
		return costTime;
	}

	public void setCostTime(int costTime) {
		this.costTime = costTime;
	}

}

那么有了主线程之后，我们还需要子线程，来下载各自负责下载的部分文件：

/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package org.app.download.component;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Serializable;
import java.net.URL;
import java.net.URLConnection;

/**
 * 
 * @Title: DLThread.java
 * @Description: 下载线程类
 * @Package org.app.download.component
 * @author [email protected]
 * @date 2012-8-1
 * @version V1.0
 * 
 */
public class DLThread extends Thread implements Serializable {
	private static final long serialVersionUID = -3317849201046281359L;
	// 　缓冲字节
	private static int BUFFER_SIZE = 8096;
	// 当前任务对象
	transient private DLTask dlTask;
	// 任务ID
	private int id;
	// URL下载地址
	private URL url;
	// 开始下载点
	private int startPos;
	// 结束下载点
	private int endPos;
	// 当前下载点
	private int curPos;
	// 读入字节
	private long readByte;
	// 文件
	transient private File file;
	// 当前线程是否下载完成
	private boolean finished;
	// 是否是新建下载任务
	private boolean isNewThread;

	public DLThread(DLTask dlTask, int id, int startPos, int endPos) {
		this.dlTask = dlTask;
		this.id = id;
		this.url = dlTask.getUrl();
		this.curPos = this.startPos = startPos;
		this.endPos = endPos;
		this.file = dlTask.getFile();
		finished = false;
		readByte = 0;
	}

	public void run() {
		System.out.println("Tread - " + id + " start......");
		BufferedInputStream bis = null;
		RandomAccessFile fos = null;
		byte[] buf = new byte[BUFFER_SIZE];
		URLConnection con = null;
		try {
			con = url.openConnection();
			con.setAllowUserInteraction(true);
			if (isNewThread) {
				con.setRequestProperty("Range", "bytes=" + startPos + "-" + endPos);
				fos = new RandomAccessFile(file, "rw");
				fos.seek(startPos);
			} else {
				con.setRequestProperty("Range", "bytes=" + curPos + "-" + endPos);
				fos = new RandomAccessFile(dlTask.getFile(), "rw");
				fos.seek(curPos);
			}
			bis = new BufferedInputStream(con.getInputStream());
			while (curPos < endPos) {
				int len = bis.read(buf, 0, BUFFER_SIZE);
				if (len == -1) {
					break;
				}

				fos.write(buf, 0, len);
				curPos = curPos + len;
				if (curPos > endPos) {
					// 获取正确读取的字节数
					readByte += len - (curPos - endPos) + 1;
				} else {
					readByte += len;
				}
			}
			System.out.println("Tread - " + id + " Has the download is complete!");
			this.finished = true;
			bis.close();
			fos.close();
		} catch (IOException ex) {
			ex.printStackTrace();
			throw new RuntimeException(ex);
		}
	}

	public boolean isFinished() {
		return finished;
	}

	public long getReadByte() {
		return readByte;
	}

	public void setDlTask(DLTask dlTask) {
		this.dlTask = dlTask;
	}
}

为了达到文件续传的目的，我们要把主线程类的相关信息写到磁盘上，待文件下载完之后就可以删除这个文件，这样我们就需要一个记录器，同样也是线程，

在主线程中执行，隔3秒保存主线程类的相关信息，序列化到磁盘上：

package org.app.download.component;

import java.io.File;
import java.math.BigDecimal;

import org.app.download.util.DownUtils;
import org.app.download.util.FileOperation;

/**
 * 
 * @Title: DLListener.java
 * @Description: 保存当前任务的及时信息
 * @Package org.app.download.component
 * @author [email protected]
 * @date 2012-8-1
 * @version V1.0
 * 
 */
public class DLListener extends Thread {
	// 当前下载任务
	private DLTask dlTask;
	// 当前记录器(保存当前任务的下载对象，用于任务恢复)
	private Recorder recoder;

	DLListener(DLTask dlTask) {
		this.dlTask = dlTask;
		this.recoder = new Recorder(dlTask);
	}

	@Override
	public void run() {
		int i = 0;
		BigDecimal completeTot = null;
		long start = System.currentTimeMillis();
		long end = start;

		while (!dlTask.isComplete()) {
			i++;
			String percent = dlTask.getCurPercent();

			completeTot = new BigDecimal(dlTask.getCompletedTot());

			end = System.currentTimeMillis();
			if (end - start > 1000) {
				BigDecimal pos = new BigDecimal(((end - start) / 1000) * 1024);
				System.out.println("Speed :" + completeTot.divide(pos, 0, BigDecimal.ROUND_HALF_EVEN) + "k/s   " + percent + "% completed. ");
			}
			recoder.record();
			try {
				sleep(3000);
			} catch (InterruptedException ex) {
				ex.printStackTrace();
				throw new RuntimeException(ex);
			}

		}

		// 计算下载花费时间
		int costTime = +(int) ((System.currentTimeMillis() - start) / 1000);
		dlTask.setCostTime(costTime);
		String time = DownUtils.changeSecToHMS(costTime);

		// 对文件重命名
		dlTask.getFile().renameTo(new File(dlTask.getFilename()));
		System.out.println("Download finished. " + time);

		// 删除记录对象状态的文件
		String tskFileName = dlTask.getFilename() + ".tsk";
		try {
			FileOperation.delete(tskFileName);
		} catch (Exception e) {
			System.out.println("Delete tak file fail!");
			e.printStackTrace();
		}
	}
}

最后我们要做的就是启动main方法创建下载的文件夹，判断是否下载过，调用主线程等等：

package org.app.download.core;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.app.download.component.DLTask;
import org.app.download.util.FileOperation;

/**
 * 下载核心引擎类
 * 
 * @Title: Engine.java
 * @Description: org.app.download.core
 * @Package org.app.download.core
 * @author [email protected]
 * @date 2012-7-31
 * @version V1.0
 * 
 */
public class EngineCore {
	// 下载最大线程数
	private static final int MAX_DLINSTANCE_QUT = 10;
	// 下载任务对象
	private DLTask[] dlTask;
	// 线程池
	public static ExecutorService pool = Executors.newCachedThreadPool();

	public DLTask[] getDlTask() {
		return dlTask;
	}

	public void setDlTask(DLTask[] dlInstance) {
		this.dlTask = dlInstance;
	}

	/**
	 * 创建新下载任务
	 * 
	 * @param threadQut
	 *            线程数
	 * @param url
	 *            下载地址
	 * @param path
	 *            文件存放地址
	 * @param filename
	 *            文件名
	 */
	public void createDLTask(int threadQut, String url, String filePath) {
		DLTask task = new DLTask(threadQut, url, filePath);
		pool.execute(task);
	}

	/**
	 * 断点续传下载任务
	 * 
	 * @param threadQut
	 *            线程数
	 * @param url
	 *            下载地址
	 * @param path
	 *            文件存放地址
	 * @param filename
	 *            文件名
	 */
	public void resumeDLTask(int threadQut, String url, String path) {
		ObjectInputStream in = null;
		try {
			in = new ObjectInputStream(new FileInputStream(path + ".tsk"));
			DLTask task = (DLTask) in.readObject();
			pool.execute(task);
		} catch (ClassNotFoundException ex) {
			Logger.getLogger(EngineCore.class.getName()).log(Level.SEVERE, null, ex);
		} catch (IOException ex) {
			Logger.getLogger(EngineCore.class.getName()).log(Level.SEVERE, null, ex);
		} finally {
			try {
				in.close();
			} catch (IOException ex) {
				Logger.getLogger(EngineCore.class.getName()).log(Level.SEVERE, null, ex);
			}
		}
	}

	/**
	 * 启动下载
	 * 
	 * @param url
	 *            下载地址
	 * @param path
	 *            本地存放目录
	 * @param threadQut
	 *            线程数量
	 * @throws Exception
	 */
	public void startDownLoad(String url, String path, int threadQut) throws Exception {
		// 创建文件夹
		FileOperation.createFolder(path);
		// 获取当前文件中所有的文件名
		Map<String, String> map = FileOperation.listFile(path);
		// 查看下载文件是否已经下载过
		String filePath = existFile(path + File.separator + url.substring(url.lastIndexOf("/") + 1, url.length()));
		// 保存下载对象的文件名
		String takFileName = filePath.substring(filePath.lastIndexOf("\\") + 1, filePath.length()) + ".tsk";
		boolean isexist = false;
		for (Map.Entry<String, String> key : map.entrySet()) {
			if (takFileName.equals(key.getKey())) {
				isexist = true;
			}
		}

		if (isexist) {
			System.out.println("Restore download task - taskName is " + takFileName);
			resumeDLTask(threadQut, url, filePath);
		} else {
			System.out.println("start downloading task -taskName is " + takFileName);
			createDLTask(threadQut, url, filePath);
		}
	}

	/**
	 * 检测文件是否存在
	 */
	private String existFile(String filePath) {
		File file = new File(filePath);
		int fileCnt = 1;
		while (file.exists()) {
			String fileFirst = filePath.substring(0, filePath.lastIndexOf(".")) + "(" + fileCnt + ")";
			String fileSecond = filePath.substring(filePath.lastIndexOf("."), filePath.length());
			filePath = fileFirst + fileSecond;
			fileCnt++;
			filePath = existFile(filePath);
			break;
		}
		return filePath;
	}

	/**
	 * 主程序入口
	 * 
	 * @param args
	 * @throws Exception
	 */
	public static void main(String[] args) throws Exception {
		EngineCore engine = new EngineCore();
		try {
			engine.startDownLoad(args[0], args[1], Integer.parseInt(args[2]));
		} catch (ArrayIndexOutOfBoundsException e) {
			engine.startDownLoad(args[0], args[1], MAX_DLINSTANCE_QUT);
		}
	}
}

最后要说的是，我借鉴了网上一哥们的代码，但是下载下来的文件全是破损，因为那哥们没有算好文件的字节，导致线程在现在的时候丢失了字节，所以文件破损，

打个比喻：假如文件有899个字节，用10线程下载 ，每个线程就是要下载89.9个字节，因为字节是整数，用int存放的话就是89个字节，每个线程丢失了0.9个字节，

所以文件就破损了。个人的解决方法就是取模，总字节减去取模的字节，让第十个线程多分配几个下载字节。

最重要的请大神不要喷我，本人纯粹自娱自乐。3Q.

不知道csdn怎样在文章中打包下载，这里贴出我google code里面的下载地址。里面还有一个生成mybatis代码的工具。

地址：http://code.google.com/p/mybatis-generator/downloads/list