java编写HDFS操作工具类

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/qq_29726869/article/details/82460751

package com.rlt.utils;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import spire.algebra.Bool;

//HDFS文件操作工具类
public class HdfsUtil {
    ArrayList<String> list = new ArrayList<String>();

    //在hdfs系统上创建目录
    public static Boolean createDir(String args) throws IOException {
        FileSystem fs = null;
        try {
            Configuration conf = new Configuration();
            Path path = new Path(args);
            //设置hdfs指定用户名root,否则没有权限访问hdfs
            System.setProperty("HADOOP_USER_NAME", "root");
            fs = path.getFileSystem(conf);
            fs.create(path);
            fs.close();
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }finally {
            fs.close();
        }
        return true;
    }

    //在hdfs系统上删除指定目录或文件
    public static Boolean deleteDir(String args) throws IOException {
        FileSystem fs = null;
        try {
            Configuration conf = new Configuration();
            Path path = new Path(args);
            //设置hdfs指定用户名root,否则没有权限访问hdfs
            System.setProperty("HADOOP_USER_NAME", "root");
             fs = path.getFileSystem(conf);
            fs.delete(path);
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }finally {
           fs.close();
        }
        return true;
    }

    //写文件
    public static Boolean createFile (String args) throws IOException {
        FileSystem fs = null;
        FSDataOutputStream out = null;
        try {
            Configuration conf = new Configuration();
            Path path = new Path(args);
            //设置hdfs指定用户名root,否则没有权限访问hdfs
            System.setProperty("HADOOP_USER_NAME", "root");
            fs = path.getFileSystem(conf);

            out = fs.create(path);
            out.writeUTF("da jia hao,cai shi zhen de hao!");
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }finally {
            out.close();
            fs.close();
        }
        return true;
    }

    //读文件
    public static Boolean readFile(String args) throws IOException {
        FileSystem fs = null;
        FSDataInputStream is = null;
        try {
            Configuration conf = new Configuration();
            Path path = new Path(args);
            //设置hdfs指定用户名root,否则没有权限访问hdfs
            System.setProperty("HADOOP_USER_NAME", "root");
            fs = path.getFileSystem(conf);

            if(fs.exists(path)){
                is = fs.open(path);
                FileStatus status = fs.getFileStatus(path);
                byte[] buffer = new byte[Integer.parseInt(String.valueOf(status.getLen()))];
                is.readFully(0, buffer);

                System.out.println(buffer.toString());
            }
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }finally {
            is.close();
            fs.close();
        }
        return true;
    }

    //上传文件到HDFS上
    public static Boolean uploadFile(String srcPath,String dstPath) throws IOException {
        FileSystem fs = null;
        try {
            Configuration conf = new Configuration();
            Path path = new Path(dstPath);
            //设置hdfs指定用户名root,否则没有权限访问hdfs
            System.setProperty("HADOOP_USER_NAME", "root");
            fs = path.getFileSystem(conf);

            Path src = new Path(srcPath);       //本地文件
            Path dst = new Path(dstPath);       //上传目录
            //上传本地文件到hdfs上,hdfs不存在指定目录则创建该目录
            fs.copyFromLocalFile(src, dst);

        }catch (Exception e) {
            e.printStackTrace();
            return false;
        }finally {
            fs.close();
        }
        return true;
    }

    //获取给定目录下的所有子目录以及子文件
    public List<String> getFileList(String args)throws IOException {
        FileSystem fs = null;
        try {
            Configuration conf = new Configuration();
            Path path = new Path(args);
            //设置hdfs指定用户名root,否则没有权限访问hdfs
            System.setProperty("HADOOP_USER_NAME", "root");
            fs = path.getFileSystem(conf);

            this.getFile(path,fs);

        } catch (Exception e) {
            e.printStackTrace();
            return null;
        }finally {
            fs.close();
        }
        return list;
    }

    public void getFile(Path path,FileSystem fs) throws IOException {

        FileStatus[] fileStatus = fs.listStatus(path);
        for(int i=0;i<fileStatus.length;i++){
            if(fileStatus[i].isDir()){
                Path p = new Path(fileStatus[i].getPath().toString());
                getFile(p,fs);
            }else{
                list.add(fileStatus[i].getPath().toString());
            }
        }
    }


    //查找某个文件在HDFS集群的位置
    public static void getFilePath(String args)throws IOException {
        FileSystem fs = null;
        try {
            Configuration conf = new Configuration();
            Path path = new Path(args);
            //设置hdfs指定用户名root,否则没有权限访问hdfs
            System.setProperty("HADOOP_USER_NAME", "root");
            fs = path.getFileSystem(conf);

            FileStatus status = fs.getFileStatus(path);
            BlockLocation[] locations = fs.getFileBlockLocations(status, 0, status.getLen());

            int length = locations.length;
            for(int i=0;i<length;i++){
                String[] hosts = locations[i].getHosts();
                System.out.println("block_" + i + "_location:" + hosts[i]);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            fs.close();
        }
    }




    //HDFS集群上所有节点名称信息
    public static void getHDFSNode(String args) throws IOException{
        FileSystem fs = null;
        DistributedFileSystem dfs = null;
        try {
            Configuration conf = new Configuration();
            Path path = new Path(args);
            //设置hdfs指定用户名root,否则没有权限访问hdfs
            System.setProperty("HADOOP_USER_NAME", "root");
            fs = path.getFileSystem(conf);
            dfs = (DistributedFileSystem)fs;
            DatanodeInfo[] dataNodeStats = dfs.getDataNodeStats();
            for(int i=0;i<dataNodeStats.length;i++){
                System.out.println("DataNode_" + i + "_Node:" + dataNodeStats[i].getHostName());
            }
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            dfs.close();
            fs.close();
        }
    }

    public static void main(String[] args) throws IOException {
//        createDir("hdfs://master:9000/dict");
//        deleteDir("hdfs://master:9000/test.txt");
//          createFile("hdfs://master:9000/test.txt");
//          readFile("hdfs://master:9000/test.txt");
//          uploadFile("C:/Users/zixiang/Desktop/sql.txt","hdfs://master:9000/test");
//            HdfsUtil hdfsUtil = new HdfsUtil();
//            ArrayList<String> list = (ArrayList<String>) hdfsUtil.getFileList("hdfs://master:9000/sougoudict");
//            for(String i:list){
//                System.out.println(i);
//            }

        getFilePath("hdfs://master:9000/test");
        getHDFSNode("hdfs://master:9000");
    }
}