一、介绍
阿里云对象存储服务（Object Storage Service，简称 OSS），是阿里云提供的海量、安全、低成本、高可靠的云存储服务。它具有与平台无关的RESTful API接口，能够提供99.999999999%（11个9）的数据可靠性和99.95%的服务可用性。您可以在任何应用、任何时间、任何地点存储和访问任意类型的数据。
二、准备工作
1.购买阿里云服务
2.基本概念：bucket是一个阿里云的存储空间，上传文件到OSS被称为Object,object可以是虚拟文件夹，以/结尾就可以了，虚拟文件夹支持上传以及下载
阿里云上传后是不会返回路径给你的，路径有一定的规则：https://bucketName.endPoint/ke
三、上传文件
阿里云提供许多种上传文件的方式，java-sdk路径：https://help.aliyun.com/document_detail/31817.html?spm=a2c4g.11186623.6.539.NRKRNu
在项目中需要上传视频，估采用分片断点上传方式，这个方式步骤：
1.初始化一个分片上传任务（InitiateMultipartUpload）
2.逐个或并行上传分片（UploadPart）
3.完成分片上传（CompleteMultipartUpload）或取消分片上传(AbortMultipartUpload)
代码实现主要是在2的步骤使用多线程并行上传，使用Executors线程池框架，项目使用spring boot框架开发：
1.分片上传类：这里特别说明下：由于第2步使用多线程并发上传分片，而第3步的需要第2步返回来的参数，在线程的shutdown()后必须加上：
//阻塞，直到关闭后所有任务都已完成执行 pool.awaitTermination(9223372036854775807L, TimeUnit.SECONDS);否则会出现第2步还没有执行完，第3步就开始执行，到时上传失
/**
* Author: hezishan
* Date: 2018/4/27.
* Description: 分片上传类
* 1.初始化一个分片上传任务（InitiateMultipartUpload）
* 2.逐个或并行上传分片（UploadPart）
* 3.完成分片上传（CompleteMultipartUpload）或取消分片上传(AbortMultipartUpload)
**/
@Service
public class MultipartUpload {
public static final Logger LOGGER = Logger.getLogger(OSSUploadFile.class);
@Autowired
private OSSConfiguration ossConfiguration;
/**
* 1.初始化一个Multi-part upload请求
* @param client
* @param bucketName
* @param key
* @return
* @throws OSSException
* @throws ClientException
*/
public String initMultipartUpload(OSSClient client,String bucketName, String key) throws OSSException,ClientException{
InitiateMultipartUploadRequest initUploadRequest = new InitiateMultipartUploadRequest(bucketName, key);
InitiateMultipartUploadResult initResult = client.initiateMultipartUpload(initUploadRequest);
String uploadId = initResult.getUploadId();
return uploadId;
}
/**
* 2.逐个或并行上传分片
* @param ossClient
* @param bucketName
* @param key
* @param uploadFile
* @param sourcePath
* @param pool
* @return ResponseEntity
* @throws Exception
*/
public ResponseEntity uploadMultipartFile(OSSClient ossClient, String bucketName, String key, File uploadFile,String sourcePath,ExecutorService pool) throws Exception{
ResponseEntity responseEntity=new ResponseEntity();
//自定义的每个上传分块大小
Integer partSize=ossConfiguration.getUploadPartSize();
//需要上传的文件分块数
int partCount=FileUtils.calPartCount(uploadFile,partSize);
//分片上传唯一标识
String uploadId="";
//序列化的文件路径
String serializationFilePath=sourcePath+".up.temp";
boolean isSerializationFile=false;
//子线程池的线程对象封装类（用于序列化的）
UploadPartObj uploadPartObj=null;
//若存在上传失败留下的序列化文件则反序列化对象
if(new File(serializationFilePath).exists()){
uploadPartObj = (UploadPartObj)ObjectSerializableUtil.load(serializationFilePath);
isSerializationFile = true;
}
//序列化文件不存在，分配分块给子线程池线程对象
if(uploadPartObj==null||!isSerializationFile){
uploadPartObj = new UploadPartObj();
try {
//1.初始化MultipartUpload 返回uploadId
uploadId = initMultipartUpload(ossClient, bucketName, key);
} catch (OSSException |ClientException e) {
LOGGER.error("[MultipartUpload.class:uploadMultipartFile]:"+e.getMessage());
responseEntity.setResponseCode(ResponseCode.OSS_SUBMIT_ERROR);
responseEntity.setReponseMsg("OSS提交出错");
return responseEntity;
}
for (int i = 0; i < partCount ; i++) {
long start = partSize * i;
long curPartSize = partSize < uploadFile.length() - start ?
partSize : uploadFile.length() - start;
//2.构造上传线程，UploadPartThread是执行每个分块上传任务的线程（使用多线程并行上传）
uploadPartObj.getUploadPartThreads().add(new UploadPartThread(ossClient, bucketName, key,uploadFile, uploadId, i + 1,partSize * i, curPartSize));
}
}
try {
int i = 0;
//upload方法提交分块上传线程至子线程池上传，while循环用于上传失败重复上传，Constant.RETRY定义重复次数
while (upload(uploadPartObj,serializationFilePath,pool).isResult()==false) {
if(++i == ossConfiguration.getRetry()){
break;
}
}
} catch (Exception e) {
LOGGER.info("[MultipartUpload.class:uploadMultipartFile]:" + e.getMessage());
responseEntity.setResponseCode(ResponseCode.THREAD_ERROR);
responseEntity.setReponseMsg("线程出错");
return responseEntity;
}
if(!uploadPartObj.isResult()){
responseEntity.setResponseCode(ResponseCode.NETWORK_ERROR);
responseEntity.setReponseMsg("网络出错");
return responseEntity;
}
try {
//3.完成一个multi-part请求。
completeMultipartUpload(ossClient, bucketName, key, uploadPartObj);
} catch (Exception e) {
LOGGER.info("[MultipartUpload.class:uploadMultipartFile]:" + e.getMessage());
ObjectSerializableUtil.save(uploadPartObj,serializationFilePath);
responseEntity.setResponseCode(ResponseCode.OSS_SUBMIT_ERROR);
responseEntity.setReponseMsg("OSS提交出错");
return responseEntity;
}
String responseUrl=OSSUtils.generateResponseUrl(bucketName,ossConfiguration.getEndPoint(),key);
responseEntity.setResponseCode(ResponseCode.SUCCESS);
responseEntity.setReponseMsg("成功");
responseEntity.setResponseData(responseUrl);
return responseEntity;
}
/**
* 3.完成一个multi-part请求
* @param client
* @param bucketName
* @param key
* @param uploadPartObj
*/
public void completeMultipartUpload(OSSClient client, String bucketName, String key,UploadPartObj uploadPartObj){
List<PartETag> eTags = new ArrayList<PartETag>();
for (UploadPartThread uploadPartThread : uploadPartObj.getUploadPartThreads()) {
eTags.add(new PartETag(uploadPartThread.getMyPartETag().getPartNumber(),uploadPartThread.getMyPartETag().geteTag()));
}
//为part按partnumber排序
Collections.sort(eTags, new Comparator<PartETag>(){
@Override
public int compare(PartETag arg0, PartETag arg1) {
PartETag part1= arg0;
PartETag part2= arg1;
return part1.getPartNumber() - part2.getPartNumber();
}
});
CompleteMultipartUploadRequest completeMultipartUploadRequest =
new CompleteMultipartUploadRequest(bucketName, key, uploadPartObj.getUploadPartThreads().get(0).getUploadId(), eTags);
client.completeMultipartUpload(completeMultipartUploadRequest);
}
/**
* 多线程上传单个文件
* @param uploadPartObj
* @param serializationFilePath
* @return
*/
private UploadPartObj upload(UploadPartObj uploadPartObj,String serializationFilePath,ExecutorService pool){
try {
uploadPartObj.setResult(true);
//向子线程池中submit单个文件所有分块上传线程
for (int i=0;i<uploadPartObj.getUploadPartThreads().size();i++) {
if (uploadPartObj.getUploadPartThreads().get(i).getMyPartETag() == null) {
pool.submit(uploadPartObj.getUploadPartThreads().get(i));
}
}
//shutdown子线程池，池内所上传任务执行结束后停止当前线程池
pool.shutdown();
while (!pool.isTerminated()) {
//循环检查线程池，同时在此序列化uploadPartObj
ObjectSerializableUtil.save(uploadPartObj,serializationFilePath);
pool.awaitTermination(ossConfiguration.getSerializationTime(), TimeUnit.SECONDS);
}
//判断上传结果
for (UploadPartThread uploadPartThread: uploadPartObj.getUploadPartThreads()) {
if(uploadPartThread.getMyPartETag()==null) {
uploadPartObj.setResult(false);
}
}
//上传成功 删除序列化文件
if (uploadPartObj.isResult()==true){
ObjectSerializableUtil.delSerlzFile(serializationFilePath);
}
} catch (Exception e) {
LOGGER.info("[MultipartUpload.class:upload]:"+e.getMessage());
}
return uploadPartObj;
}
}
2.上传文件类：支持多文件并发上传，每个文件支持并发上传分片
/**
* Author: hezishan
* Date: 2018/4/26.
* Description: 上传文件
**/
@Service
public class OSSUploadFile implements Callable<ResponseEntity> {
public static final Logger LOGGER = Logger.getLogger(OSSUploadFile.class);
@Autowired
private ObjectManager objectManager;
@Autowired
private MultipartUpload multipartUpload;
@Autowired
private OSSConfiguration configuration;
//外部线程池
public static ExecutorService uploadMainPool=null;
static{
OSSConfiguration oss =(OSSConfiguration) SpringUtils.getBean("OSSConfiguration");
uploadMainPool=Executors.newFixedThreadPool(oss.getConcurrentFileNumber(), new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread s = Executors.defaultThreadFactory().newThread(r);
s.setDaemon(true);
return s;
}
});
}
/**
* 内部线程池
*/
private ExecutorService pool;
/**
* 上传路径
*/
private String sourcePath;
/**
* buckername
*/
private String bucketName;
/**
* 云端存储路径
*/
private String key;
/**
* 执行当前线程
* @return
*/
public ResponseEntity uploadFile(String sourcePath,Integer uploadType){
//实例化单个文件上传线程
pool=Executors.newFixedThreadPool(configuration.getSingleFileConcrrentThreads());
this.sourcePath=sourcePath;
this.bucketName=configuration.getBucketName();
this.key=OSSUtils.generateKeyUrl(uploadType,sourcePath);
ResponseEntity responseEntity=new ResponseEntity();
//向uploadMainPool中submit当前线程
Future<ResponseEntity> result=uploadMainPool.submit(this);
try{
responseEntity=result.get();
}catch (Exception e){
LOGGER.info("[OSSUploadFile.class:uploadFile]:"+e);
responseEntity.setResponseCode(ResponseCode.ERROR);
responseEntity.setReponseMsg("执行上传文件出错");
return responseEntity;
}
return responseEntity;
}
/**
* 上传文件
* @return
*/
@Override
public ResponseEntity call() throws Exception {
ResponseEntity responseEntity=new ResponseEntity();
OSSClient ossClient=OSSClientFactory.getInstance();
File uploadFile=new File(sourcePath);
if(!uploadFile.exists()){
LOGGER.info("[OSSUploadFile.class:call]:"+sourcePath);
responseEntity.setResponseCode(ResponseCode.FILE_NOT_FOUND_ERROR);
responseEntity.setReponseMsg("文件未找到");
return responseEntity;
}
int re=objectManager.createBucket(ossClient,bucketName);
if(re==ResponseCode.NEW_BUCKET_ERROR){
responseEntity.setResponseCode(ResponseCode.NEW_BUCKET_ERROR);
responseEntity.setReponseMsg("创建bucket错误");
return responseEntity;
}
//分片续点上传
responseEntity=multipartUpload.uploadMultipartFile(ossClient,bucketName,key,uploadFile,sourcePath,pool);
pool=null;
return responseEntity;
}
4.使用线程安全的单例模式获取OSSClient
/** * Author: hezishan * Date: 2018/4/25. * Description: OSSClientFactory OSSClient工厂类 **/ @Component public class OSSClientFactory { private static OSSClient ossClient = null; /** * 获取一个ossclient对象 * @return */ public static OSSClient getInstance() { try{ synchronized (OSSClientFactory.class) { if (ossClient == null) { //加载配置文件 OSSConfiguration ossConfiguration =(OSSConfiguration) SpringUtils.getBean("OSSConfiguration"); if(ossConfiguration!=null){ ossClient = new OSSClient(ossConfiguration.getEndPoint(),ossConfiguration.getAccessKeyId(), ossConfiguration.getAccessKeySecret()); } } } }catch (Exception e){ e.printStackTrace(); } return ossClient; } }
5.其他工具类：
MyPartETag：是对步骤2返回的数据进行封装，并在第3步使用：
/**
* Author: hezishan
* Date: 2018/4/26.
* Description: MyPartETag类
* 用于保存分片上传的PartETag
**/
public class MyPartETag implements Serializable {
private static final long serialVersionUID = 1L;
private int partNumber;
private String eTag;
public MyPartETag(PartETag partETag ) {
super();
this.partNumber = partETag.getPartNumber();
this.eTag = partETag.getETag();
}
public int getPartNumber() {
return partNumber;
}
public void setPartNumber(int partNumber) {
this.partNumber = partNumber;
}
public String geteTag() {
return eTag;
}
public void seteTag(String eTag) {
this.eTag = eTag;
}
}
UploadPartObj
public class UploadPartObj implements Serializable {
private static final long serialVersionUID = 1L;
List<UploadPartThread> uploadPartThreads = Collections.synchronizedList(new ArrayList<UploadPartThread>());
boolean result = true;
public List<UploadPartThread> getUploadPartThreads() {
return uploadPartThreads;
}
public void setUploadPartThreads(List<UploadPartThread> uploadPartThreads) {
this.uploadPartThreads = uploadPartThreads;
}
public boolean isResult() {
return result;
}
public void setResult(boolean result) {
this.result = result;
}
}
UploadPartThread
/**
* Author: hezishan
* Date: 2018/4/26.
* Description:上传的分片线程类
**/
public class UploadPartThread implements Callable<UploadPartThread>,Serializable {
private static final long serialVersionUID = 1L;
public static final Logger LOGGER = Logger.getLogger(UploadPartThread.class);
private OSSClient ossClient;
private File uploadFile;
private String bucket;
private String object;
private long start;
private long size;
private int partId;
private String uploadId;
private MyPartETag myPartETag;
public UploadPartThread(OSSClient ossClient, String bucket, String object,
File uploadFile, String uploadId, int partId,
long start, long partSize) {
this.ossClient=ossClient;
this.uploadFile = uploadFile;
this.bucket = bucket;
this.object = object;
this.start = start;
this.size = partSize;
this.partId = partId;
this.uploadId = uploadId;
}
@Override
public UploadPartThread call() {
InputStream in = null;
try {
in = new FileInputStream(uploadFile);
in.skip(start);
UploadPartRequest uploadPartRequest = new UploadPartRequest();
uploadPartRequest.setBucketName(bucket);
uploadPartRequest.setKey(object);
uploadPartRequest.setUploadId(uploadId);
uploadPartRequest.setInputStream(in);
uploadPartRequest.setPartSize(size);
uploadPartRequest.setPartNumber(partId);
//MyPartETag是对uploadPartResult.getPartETag()的返回值PartETag的封装，主要是为了能序列化PartETag，MyPartETag仅比PartETag多实现了Serializable接口
UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest);
myPartETag = new MyPartETag(uploadPartResult.getPartETag());
} catch (Exception e) {
LOGGER.error("[UploadPartThread.class:UploadPartThread]:"+e.getMessage());
} finally {
if (in != null){
try {
in.close();
} catch (Exception e) {
LOGGER.error("[UploadPartThread.class:UploadPartThread]:"+e.getMessage());
}
}
}
return this;
}
public String getUploadId() {
return uploadId;
}
public void setUploadId(String uploadId) {
this.uploadId = uploadId;
}
public MyPartETag getMyPartETag() {
return myPartETag;
}
public void setMyPartETag(MyPartETag myPartETag) {
this.myPartETag = myPartETag;
}
}