- import java.io.ByteArrayInputStream;
- import java.io.IOException;
- import java.io.InputStream;
- import java.io.OutputStream;
- import java.net.URI;
- import java.util.ArrayList;
- import java.util.List;
- import org.apache.commons.lang.StringUtils;
- import org.apache.hadoop.conf.Configuration;
- import org.apache.hadoop.fs.FSDataInputStream;
- import org.apache.hadoop.fs.FSDataOutputStream;
- import org.apache.hadoop.fs.FileStatus;
- import org.apache.hadoop.fs.FileSystem;
- import org.apache.hadoop.fs.Path;
- import org.apache.zookeeper.common.IOUtils;
- /**
- * operate hdfs file or directory util class
- *
- * @author zhang
- * @since 2016-09-26
- *
- */
- public class HdfsUtils {
- public static String uri = "hdfs://192.168.1.118:9000";
- /**
- * 创建文件夹
- *
- * @param dir 文件夹路径 '/tmp/testdir'
- * @return boolean true-success, false-failed
- * @exception IOException something wrong happends when operating files
- */
- public static boolean mkdir(String dir) throws IOException {
- if (StringUtils.isBlank(dir)) {
- return false;
- }
- dir = uri + dir;
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(URI.create(dir), conf);
- if (!fs.exists(new Path(dir))) {
- fs.mkdirs(new Path(dir));
- }
- fs.close();
- return true;
- }
- /**
- * 删除文件夹
- * if dir not exists, it will throw FileNotFoundException
- *
- * @param dir要删除文件夹路径 '/tmp/testdir'
- * @return boolean true-success, false-failed
- * @exception IOException something wrong happends when operating files
- *
- */
- public static boolean deleteDir(String dir) throws IOException {
- if (StringUtils.isBlank(dir)) {
- return false;
- }
- dir = uri + dir;
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(URI.create(dir), conf);
- fs.delete(new Path(dir), true);
- fs.close();
- return true;
- }
- /**
- * 获取hdfs中指定文件夹下所有文件信息
- *
- * @param dir a folder path may like '/tmp/testdir'
- * @return List<String> list of file names
- * @throws IOException file io exception
- */
- public static List<String> listAll(String dir) throws IOException {
- if (StringUtils.isBlank(dir)) {
- return new ArrayList<String>();
- }
- dir = uri + dir;
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(URI.create(dir), conf);
- FileStatus[] stats = fs.listStatus(new Path(dir));
- List<String> names = new ArrayList<String>();
- for (int i = 0; i < stats.length; ++i) {
- if (stats[i].isFile()) {
- // regular file
- names.add(stats[i].getPath().toString());
- } else if (stats[i].isDirectory()) {
- // dir
- names.add(stats[i].getPath().toString());
- } else if (stats[i].isSymlink()) {
- // is s symlink in linux
- names.add(stats[i].getPath().toString());
- }
- }
- fs.close();
- return names;
- }
- /*
- * 上传文件到hdfs,
- * notice that the path is full like /tmp/test.txt
- * if local file not exists, it will throw a FileNotFoundException
- *
- * @param localFile local file path, may like F:/test.txt or /usr/local/test.txt
- *
- * @param hdfsFile hdfs file path, may like /tmp/dir
- * @return boolean true-success, false-failed
- *
- * @throws IOException file io exception
- */
- public static boolean uploadLocalFile2HDFS(String localFile, String hdfsFile) throws IOException {
- if (StringUtils.isBlank(localFile) || StringUtils.isBlank(hdfsFile)) {
- return false;
- }
- hdfsFile = uri + hdfsFile;
- Configuration config = new Configuration();
- FileSystem hdfs = FileSystem.get(URI.create(uri), config);
- // FSDataOutputStream create = fs.create(new Path(hdfsFile+"/"+文件名)); 获取文件输出流
- // 获取文件输出流2
- // OutputStream out = fs.create(new Path(DEST_PATH), new Progressable() {
// @Override
// public void progress() {
// System.out.println("上传完一个设定缓存区大小容量的文件!");
// }
// }); - /*byte[] buffer = new byte[1024];
int len = 0;
while((len=in.read(buffer))>0){
out.write(buffer, 0, len);
}
out.flush();
in.close();
out.close(); - */
- Path src = new Path(localFile);
- Path dst = new Path(hdfsFile);
- hdfs.copyFromLocalFile(src, dst);
- hdfs.close();
- return true;
- }
- /*
- * 在hdfs中新建文件.
- *
- * notice that the toCreateFilePath is the full path
- *
- * and write the content to the hdfs file.
- */
- /**
- * create a new file in the hdfs.
- * if dir not exists, it will create one
- *
- * @param newFile new file path, a full path name, may like '/tmp/test.txt'
- * @param content file content
- * @return boolean true-success, false-failed
- * @throws IOException file io exception
- */
- public static boolean createNewHDFSFile(String newFile, String content) throws IOException {
- if (StringUtils.isBlank(newFile) || null == content) {
- return false;
- }
- newFile = uri + newFile;
- Configuration config = new Configuration();
- FileSystem hdfs = FileSystem.get(URI.create(newFile), config);
- FSDataOutputStream os = hdfs.create(new Path(newFile));
- os.write(content.getBytes("UTF-8"));
- os.close();
- hdfs.close();
- return true;
- }
- /**
- * 删除hdfs中文件
- *
- * @param hdfsFile a full path name, may like '/tmp/test.txt'
- * @return boolean true-success, false-failed
- * @throws IOException file io exception
- */
- public static boolean deleteHDFSFile(String hdfsFile) throws IOException {
- if (StringUtils.isBlank(hdfsFile)) {
- return false;
- }
- hdfsFile = uri + hdfsFile;
- Configuration config = new Configuration();
- FileSystem hdfs = FileSystem.get(URI.create(hdfsFile), config);
- Path path = new Path(hdfsFile);
- boolean isDeleted = hdfs.delete(path, true);
- hdfs.close();
- return isDeleted;
- }
- /**
- * 获取hdfs中指定文件内容
- *
- * @param hdfsFile a full path name, may like '/tmp/test.txt'
- * @return byte[] file content
- * @throws IOException file io exception
- */
- public static byte[] readHDFSFile(String hdfsFile) throws Exception {
- if (StringUtils.isBlank(hdfsFile)) {
- return null;
- }
- hdfsFile = uri + hdfsFile;
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(URI.create(hdfsFile), conf);
- // check if the file exists
- Path path = new Path(hdfsFile);
- if (fs.exists(path)) {
- FSDataInputStream is = fs.open(path);
- // get the file info to create the buffer
- FileStatus stat = fs.getFileStatus(path);
- // create the buffer
- byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat.getLen()))];
- is.readFully(0, buffer);
- is.close();
- fs.close();
- return buffer;
- } else {
- throw new Exception("the file is not found .");
- }
- }
- /**
- * 在hdfs中指定文件中追加内容
- *
- * @param hdfsFile a full path name, may like '/tmp/test.txt'
- * @param content string
- * @return boolean true-success, false-failed
- * @throws Exception something wrong
- */
- public static boolean append(String hdfsFile, String content) throws Exception {
- if (StringUtils.isBlank(hdfsFile)) {
- return false;
- }
- if(StringUtils.isEmpty(content)){
- return true;
- }
- hdfsFile = uri + hdfsFile;
- Configuration conf = new Configuration();
- // solve the problem when appending at single datanode hadoop env
- conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
- conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
- FileSystem fs = FileSystem.get(URI.create(hdfsFile), conf);
- // check if the file exists
- Path path = new Path(hdfsFile);
- if (fs.exists(path)) {
- try {
- InputStream in = new ByteArrayInputStream(content.getBytes());
- OutputStream out = fs.append(new Path(hdfsFile));
- IOUtils.copyBytes(in, out, 4096, true);
- out.close();
- in.close();
- fs.close();
- } catch (Exception ex) {
- fs.close();
- throw ex;
- }
- } else {
- HdfsUtils.createNewHDFSFile(hdfsFile, content);
- }
- return true;
- }
- }
hdfs 文件操作
猜你喜欢
转载自blog.csdn.net/qq_38621910/article/details/80401552
今日推荐
周排行