最近闲来无事,因为公司屏蔽了迅雷软件的下载端口,所以自己写了一个下载工具。拿过来分享下。
下载网络上的文件肯定不能只用单线程下载,这样下载太慢,网速得不多合理利用。那么就应该用多线程下载和线程池调度线程。
所以我们要讲文件切分成N段下载。用到了RandomAccessFile 随机访问文件。
首先我们写一个主线程,用来管理下载的子线程:
package org.app.download.component;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.math.BigDecimal;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.Map;
import java.util.Set;
import org.app.download.core.EngineCore;
/**
*
* @Title: DLTask.java
* @Description: 本类对应一个下载任务,每个下载任务包含多个下载线程,默认最多包含十个下载线程
* @Package org.app.download.component
* @author [email protected]
* @date 2012-8-1
* @version V1.0
*
*/
public class DLTask extends Thread implements Serializable {
private static final long serialVersionUID = 126148287461276024L;
// 下载临时文件后缀,下载完成后将自动被删除
public final static String FILE_POSTFIX = ".tmp";
// URL地址
private URL url;
// 文件对象
private File file;
// 文件名
private String filename;
// 下载线程数量,用户可定制
private int threadQut;
// 下载文件长度
private int contentLen;
// 当前下载完成总数
private long completedTot;
// 下载时间计数,记录下载耗费的时间
private int costTime;
// 下载百分比
private String curPercent;
// 是否新建下载任务,可能是断点续传任务
private boolean isNewTask;
// 保存当前任务的线程
private DLThread[] dlThreads;
// 当前任务的监听器,用于即时获取相关下载信息
transient private DLListener listener;
public DLTask(int threadQut, String url, String filename) {
this.threadQut = threadQut;
this.filename = filename;
costTime = 0;
curPercent = "0";
isNewTask = true;
this.dlThreads = new DLThread[threadQut];
this.listener = new DLListener(this);
try {
this.url = new URL(url);
} catch (MalformedURLException ex) {
ex.printStackTrace();
throw new RuntimeException(ex);
}
}
@Override
public void run() {
if (isNewTask) {
newTask();
return;
}
resumeTask();
}
/**
* 恢复任务时被调用,用于断点续传时恢复各个线程。
*/
private void resumeTask() {
listener = new DLListener(this);
file = new File(filename + FILE_POSTFIX);
for (int i = 0; i < threadQut; i++) {
dlThreads[i].setDlTask(this);
EngineCore.pool.execute(dlThreads[i]);
}
EngineCore.pool.execute(listener);
}
/**
* 新建任务时被调用,通过连接资源获取资源相关信息,并根据具体长度创建线程块, 线程创建完毕后,即刻通过线程池进行调度
*
* @throws RuntimeException
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
private void newTask() throws RuntimeException {
try {
isNewTask = false;
URLConnection con = url.openConnection();
Map map = con.getHeaderFields();
Set<String> set = map.keySet();
for (String key : set) {
System.out.println(key + " : " + map.get(key));
}
contentLen = con.getContentLength();
if (contentLen <= 0) {
System.out.println("Unable to get resources length, the interrupt download process!");
return;
}
file = new File(filename + FILE_POSTFIX);
int fileCnt = 1;
while (file.exists()) {
file = new File(filename += (fileCnt + FILE_POSTFIX));
fileCnt++;
}
int subLenMore = contentLen % threadQut;
int subLen = (contentLen - subLenMore) / threadQut;
for (int i = 0; i < threadQut; i++) {
DLThread thread;
if (i == threadQut - 1) {
thread = new DLThread(this, i + 1, subLen * i, (subLen * (i + 1) - 1) + subLenMore);
} else {
thread = new DLThread(this, i + 1, subLen * i, subLen * (i + 1) - 1);
}
dlThreads[i] = thread;
EngineCore.pool.execute(dlThreads[i]);
}
EngineCore.pool.execute(listener);
} catch (IOException ex) {
ex.printStackTrace();
throw new RuntimeException(ex);
}
}
/**
* 计算当前已经完成的长度并返回下载百分比的字符串表示,目前百分比均为整数
*
* @return
*/
public String getCurPercent() {
this.completeTot();
curPercent = new BigDecimal(completedTot).divide(new BigDecimal(this.contentLen), 2, BigDecimal.ROUND_HALF_EVEN).divide(new BigDecimal(0.01), 0, BigDecimal.ROUND_HALF_EVEN).toString();
return curPercent;
}
/**
* 获取当前下载的字节
*/
private void completeTot() {
completedTot = 0;
for (DLThread t : dlThreads) {
completedTot += t.getReadByte();
}
}
/**
* 判断全部线程是否已经下载完成,如果完成则返回true,相反则返回false
*
* @return
*/
public boolean isComplete() {
boolean completed = true;
for (DLThread t : dlThreads) {
completed = t.isFinished();
if (!completed) {
break;
}
}
return completed;
}
/**
* 下载完成后重命名文件
*/
public void rename() {
this.file.renameTo(new File(filename));
}
public DLThread[] getDlThreads() {
return dlThreads;
}
public void setDlThreads(DLThread[] dlThreads) {
this.dlThreads = dlThreads;
}
public File getFile() {
return file;
}
public URL getUrl() {
return url;
}
public int getContentLen() {
return contentLen;
}
public String getFilename() {
return filename;
}
public int getThreadQut() {
return threadQut;
}
public long getCompletedTot() {
return completedTot;
}
public int getCostTime() {
return costTime;
}
public void setCostTime(int costTime) {
this.costTime = costTime;
}
}
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package org.app.download.component;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Serializable;
import java.net.URL;
import java.net.URLConnection;
/**
*
* @Title: DLThread.java
* @Description: 下载线程类
* @Package org.app.download.component
* @author [email protected]
* @date 2012-8-1
* @version V1.0
*
*/
public class DLThread extends Thread implements Serializable {
private static final long serialVersionUID = -3317849201046281359L;
// 缓冲字节
private static int BUFFER_SIZE = 8096;
// 当前任务对象
transient private DLTask dlTask;
// 任务ID
private int id;
// URL下载地址
private URL url;
// 开始下载点
private int startPos;
// 结束下载点
private int endPos;
// 当前下载点
private int curPos;
// 读入字节
private long readByte;
// 文件
transient private File file;
// 当前线程是否下载完成
private boolean finished;
// 是否是新建下载任务
private boolean isNewThread;
public DLThread(DLTask dlTask, int id, int startPos, int endPos) {
this.dlTask = dlTask;
this.id = id;
this.url = dlTask.getUrl();
this.curPos = this.startPos = startPos;
this.endPos = endPos;
this.file = dlTask.getFile();
finished = false;
readByte = 0;
}
public void run() {
System.out.println("Tread - " + id + " start......");
BufferedInputStream bis = null;
RandomAccessFile fos = null;
byte[] buf = new byte[BUFFER_SIZE];
URLConnection con = null;
try {
con = url.openConnection();
con.setAllowUserInteraction(true);
if (isNewThread) {
con.setRequestProperty("Range", "bytes=" + startPos + "-" + endPos);
fos = new RandomAccessFile(file, "rw");
fos.seek(startPos);
} else {
con.setRequestProperty("Range", "bytes=" + curPos + "-" + endPos);
fos = new RandomAccessFile(dlTask.getFile(), "rw");
fos.seek(curPos);
}
bis = new BufferedInputStream(con.getInputStream());
while (curPos < endPos) {
int len = bis.read(buf, 0, BUFFER_SIZE);
if (len == -1) {
break;
}
fos.write(buf, 0, len);
curPos = curPos + len;
if (curPos > endPos) {
// 获取正确读取的字节数
readByte += len - (curPos - endPos) + 1;
} else {
readByte += len;
}
}
System.out.println("Tread - " + id + " Has the download is complete!");
this.finished = true;
bis.close();
fos.close();
} catch (IOException ex) {
ex.printStackTrace();
throw new RuntimeException(ex);
}
}
public boolean isFinished() {
return finished;
}
public long getReadByte() {
return readByte;
}
public void setDlTask(DLTask dlTask) {
this.dlTask = dlTask;
}
}
为了达到文件续传的目的,我们要把主线程类的相关信息写到磁盘上,待文件下载完之后就可以删除这个文件,这样我们就需要一个记录器,同样也是线程,
在主线程中执行,隔3秒保存主线程类的相关信息,序列化到磁盘上:
package org.app.download.component;
import java.io.File;
import java.math.BigDecimal;
import org.app.download.util.DownUtils;
import org.app.download.util.FileOperation;
/**
*
* @Title: DLListener.java
* @Description: 保存当前任务的及时信息
* @Package org.app.download.component
* @author [email protected]
* @date 2012-8-1
* @version V1.0
*
*/
public class DLListener extends Thread {
// 当前下载任务
private DLTask dlTask;
// 当前记录器(保存当前任务的下载对象,用于任务恢复)
private Recorder recoder;
DLListener(DLTask dlTask) {
this.dlTask = dlTask;
this.recoder = new Recorder(dlTask);
}
@Override
public void run() {
int i = 0;
BigDecimal completeTot = null;
long start = System.currentTimeMillis();
long end = start;
while (!dlTask.isComplete()) {
i++;
String percent = dlTask.getCurPercent();
completeTot = new BigDecimal(dlTask.getCompletedTot());
end = System.currentTimeMillis();
if (end - start > 1000) {
BigDecimal pos = new BigDecimal(((end - start) / 1000) * 1024);
System.out.println("Speed :" + completeTot.divide(pos, 0, BigDecimal.ROUND_HALF_EVEN) + "k/s " + percent + "% completed. ");
}
recoder.record();
try {
sleep(3000);
} catch (InterruptedException ex) {
ex.printStackTrace();
throw new RuntimeException(ex);
}
}
// 计算下载花费时间
int costTime = +(int) ((System.currentTimeMillis() - start) / 1000);
dlTask.setCostTime(costTime);
String time = DownUtils.changeSecToHMS(costTime);
// 对文件重命名
dlTask.getFile().renameTo(new File(dlTask.getFilename()));
System.out.println("Download finished. " + time);
// 删除记录对象状态的文件
String tskFileName = dlTask.getFilename() + ".tsk";
try {
FileOperation.delete(tskFileName);
} catch (Exception e) {
System.out.println("Delete tak file fail!");
e.printStackTrace();
}
}
}
最后我们要做的就是启动main方法创建下载的文件夹,判断是否下载过,调用主线程等等:
package org.app.download.core;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.app.download.component.DLTask;
import org.app.download.util.FileOperation;
/**
* 下载核心引擎类
*
* @Title: Engine.java
* @Description: org.app.download.core
* @Package org.app.download.core
* @author [email protected]
* @date 2012-7-31
* @version V1.0
*
*/
public class EngineCore {
// 下载最大线程数
private static final int MAX_DLINSTANCE_QUT = 10;
// 下载任务对象
private DLTask[] dlTask;
// 线程池
public static ExecutorService pool = Executors.newCachedThreadPool();
public DLTask[] getDlTask() {
return dlTask;
}
public void setDlTask(DLTask[] dlInstance) {
this.dlTask = dlInstance;
}
/**
* 创建新下载任务
*
* @param threadQut
* 线程数
* @param url
* 下载地址
* @param path
* 文件存放地址
* @param filename
* 文件名
*/
public void createDLTask(int threadQut, String url, String filePath) {
DLTask task = new DLTask(threadQut, url, filePath);
pool.execute(task);
}
/**
* 断点续传下载任务
*
* @param threadQut
* 线程数
* @param url
* 下载地址
* @param path
* 文件存放地址
* @param filename
* 文件名
*/
public void resumeDLTask(int threadQut, String url, String path) {
ObjectInputStream in = null;
try {
in = new ObjectInputStream(new FileInputStream(path + ".tsk"));
DLTask task = (DLTask) in.readObject();
pool.execute(task);
} catch (ClassNotFoundException ex) {
Logger.getLogger(EngineCore.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(EngineCore.class.getName()).log(Level.SEVERE, null, ex);
} finally {
try {
in.close();
} catch (IOException ex) {
Logger.getLogger(EngineCore.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
/**
* 启动下载
*
* @param url
* 下载地址
* @param path
* 本地存放目录
* @param threadQut
* 线程数量
* @throws Exception
*/
public void startDownLoad(String url, String path, int threadQut) throws Exception {
// 创建文件夹
FileOperation.createFolder(path);
// 获取当前文件中所有的文件名
Map<String, String> map = FileOperation.listFile(path);
// 查看下载文件是否已经下载过
String filePath = existFile(path + File.separator + url.substring(url.lastIndexOf("/") + 1, url.length()));
// 保存下载对象的文件名
String takFileName = filePath.substring(filePath.lastIndexOf("\\") + 1, filePath.length()) + ".tsk";
boolean isexist = false;
for (Map.Entry<String, String> key : map.entrySet()) {
if (takFileName.equals(key.getKey())) {
isexist = true;
}
}
if (isexist) {
System.out.println("Restore download task - taskName is " + takFileName);
resumeDLTask(threadQut, url, filePath);
} else {
System.out.println("start downloading task -taskName is " + takFileName);
createDLTask(threadQut, url, filePath);
}
}
/**
* 检测文件是否存在
*/
private String existFile(String filePath) {
File file = new File(filePath);
int fileCnt = 1;
while (file.exists()) {
String fileFirst = filePath.substring(0, filePath.lastIndexOf(".")) + "(" + fileCnt + ")";
String fileSecond = filePath.substring(filePath.lastIndexOf("."), filePath.length());
filePath = fileFirst + fileSecond;
fileCnt++;
filePath = existFile(filePath);
break;
}
return filePath;
}
/**
* 主程序入口
*
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
EngineCore engine = new EngineCore();
try {
engine.startDownLoad(args[0], args[1], Integer.parseInt(args[2]));
} catch (ArrayIndexOutOfBoundsException e) {
engine.startDownLoad(args[0], args[1], MAX_DLINSTANCE_QUT);
}
}
}
最后要说的是,我借鉴了网上一哥们的代码,但是下载下来的文件全是破损,因为那哥们没有算好文件的字节,导致线程在现在的时候丢失了字节,所以文件破损,
打个比喻:假如文件有899个字节,用10线程下载 ,每个线程就是要下载89.9个字节,因为字节是整数,用int存放的话就是89个字节,每个线程丢失了0.9个字节,
所以文件就破损了。个人的解决方法就是取模,总字节减去取模的字节,让第十个线程多分配几个下载字节。
最重要的请大神不要喷我,本人纯粹自娱自乐。3Q.
不知道csdn怎样在文章中打包下载,这里贴出我google code里面的下载地址。里面还有一个生成mybatis代码的工具。
地址:http://code.google.com/p/mybatis-generator/downloads/list