装饰者模式封装对象流
import org.apache.hadoop.fs.FSDataOutputStream;
import java.io.IOException;
/**
*装饰流
*/
public class MyFSDataOutputStream extends FSDataOutputStream{
private String path ;
private FSDataOutputStream out ;
private HDFSOutputStreamPool pool ;
public MyFSDataOutputStream(String path , FSDataOutputStream out, HDFSOutputStreamPool pool) throws IOException{
super(null);
this.out = out ;
this.pool = pool ;
}
public void close(){
try{
out.close();
}
catch(Exception e){
e.printStackTrace();
}
}
public void hflush() throws IOException {
out.hflush();
}
public void write(byte[] b) throws IOException {
out.write(b);
}
public void hsync() throws IOException {
out.hsync();
}
/**
* 回收
*/
public void release(){
pool.putBack(path, this);
}
}
对象池
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.util.HashMap;
import java.util.Map;
/**
* 输出流池
*/
public class HDFSOutputStreamPool {
private FileSystem fs;
//存放的所有的输出流
private Map<String, FSDataOutputStream> pool = new HashMap<String, FSDataOutputStream>();
private static HDFSOutputStreamPool instance;
private HDFSOutputStreamPool() {
try {
Configuration conf = new Configuration();
fs = FileSystem.get(conf);
} catch (Exception e) {
e.printStackTrace();
}
}
public static HDFSOutputStreamPool getInstance() {
if (instance == null) {
instance = new HDFSOutputStreamPool();
}
return instance;
}
/**
* 通过路径得到对应的输出流
*/
public synchronized FSDataOutputStream takeOutputStream(String path) {
try{
FSDataOutputStream out = pool.remove(path);
if(out == null){
Path p = new Path(path);
if(!fs.exists(p)){
fs.createNewFile(p);
}
out = fs.append(p);
}
//转换成自己的流
return new MyFSDataOutputStream(path,out,this) ;
}
catch(Exception e){
e.printStackTrace();
}
return null ;
}
/**
* 回收流
*/
public synchronized void putBack(String path,FSDataOutputStream out){
pool.put(path,out) ;
}
/**
* 释放池子
*/
public synchronized void releasePool(){
try{
for(FSDataOutputStream o : pool.values()){
o.close();
}
pool.clear();
System.out.println("池子释放了!!!");
}
catch(Exception e){
e.printStackTrace();
}
}
}
关闭对象流,关闭对象流的对象
import java.util.TimerTask;
import org.apache.spark.ml.feature.Tokenizer;
/**
* 关闭线程池
*/
public class CloseFSOuputStreamTask extends TimerTask{
public void run() {
HDFSOutputStreamPool pool = HDFSOutputStreamPool.getInstance();
pool.releasePool();
}
}
定时关闭对象池
//开启定时器任务,周期性关闭流
new Timer().schedule(new CloseFSOuputStreamTask(), 0, 30000);
//hdfs消费者
new Thread(){
public void run() {
HDFSRawConsumer consumer = new HDFSRawConsumer();
consumer.processLog();
}
}.start();