hdfs的javaAPI开发（基础）

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.testng.annotations.Test;

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * java api的开发
 */
public class HdfsOperate {

    /**
     * 1、创建文件夹
     */
    @Test
    public void mkdirToHdfs() throws IOException {
        //创建参数对象configuration
        Configuration configuration = new Configuration();
        //如果是操作hdfs上面的文件，一定要配置fs.defaultFS
        configuration.set("fs.defaultFS","hdfs://node01:8020");
        FileSystem fileSystem = FileSystem.get(configuration);
        fileSystem.mkdirs(new Path("/fengge/dir1"));
        fileSystem.close();

    }

    /**
     * 2、文件上传
     */
    @Test
    public void uploadToHdfs() throws IOException {
        //创建参数对象configuration
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://node01:8020");
        //（1）获取客户端对象FileSystem
        FileSystem fileSystem = FileSystem.get(configuration);
        //（2）再进行文件上传
        fileSystem.copyFromLocalFile(new Path("file:///E:\\hdfs\\hello.txt"),new Path("hdfs://node01:8020/fengge/dir1"));

        fileSystem.close();


    }

    /**
     * 3、文件下载
     */
    @Test
    public void downloadFromHdfs() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://node01:8020");

        FileSystem fileSystem = FileSystem.get(configuration);

        fileSystem.copyToLocalFile(new Path("hdfs://node01:8020/fengge/dir1"),new Path("file:///E:\\hdfs\\hello2.txt"));
        fileSystem.close();
    }

    /**
     * 4、删除文件
     */
    @Test
    public void deleteFromHdfs() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://node01:8020");
        FileSystem fileSystem = FileSystem.get(configuration);

        fileSystem.deleteOnExit(new Path("hdfs://node01:8020/fengge/dir1/hell.txt"));
        fileSystem.close();
    }

    /**
     * 5、文件的重命名
     */
    @Test
    public void renameFile() throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS","hdfs://node01:8020");
        FileSystem fileSystem = FileSystem.get(configuration);
        fileSystem.rename(new Path("hdfs://node01:8020/kaikeba/dir1/hello.txt"),new Path("hdfs://node01:8020/fengge/dir1/hello3333.txt"));
        fileSystem.close();
    }

    /**
     * 6、hdfs文件相关信息
     */
    @Test
    public void testListFiles() throws IOException, URISyntaxException {
        //(1)获取文件系统
        Configuration configuration = new Configuration();
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"),configuration);
        //(2)获取文件详情
        RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"),true);

        while (listFiles.hasNext()){
            //输出详情
            LocatedFileStatus status = listFiles.next();
            //获取文件名称
            System.out.println(status.getPath().getName());
            //获取长度
            System.out.println(status.getLen());
            //获取权限
            System.out.println(status.getPermission());
            //分组
            System.out.println(status.getGroup());

            //获取存储的块信息
            BlockLocation[] blockLocations = status.getBlockLocations();
            for(BlockLocation blockLocation : blockLocations){
                //获取块存储的主机节点
                String[] hosts= blockLocation.getHosts();
                for(String host : hosts){
                    System.out.println(host);
                }
            }
        }
        fileSystem.close();
    }

    /**
     * 7、上传
     */
    @Test
    public void putFileToHdfs() throws IOException, URISyntaxException {
        //（1）获取文件系统
        Configuration configuration = new Configuration();
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"),configuration);
        //（2）创建输入流
        FileInputStream fileInputStream = new FileInputStream(new File("e:/helo.txt"));
        //（3）获取输出流
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("hdfs://node01:8020/outresult.txt"));
        //（4）流对拷
        IOUtils.copy(fileInputStream,fsDataOutputStream);
        //(5)关闭资源
        IOUtils.closeQuietly(fsDataOutputStream);
        IOUtils.closeQuietly(fileInputStream);
        fileSystem.close();

    }

    /**
     * 8、从hdfs上面下载文件
     */
    @Test
    public void downLoadFile() throws URISyntaxException, IOException {
        //(1)获取文件系统
        Configuration configuration = new Configuration();
        FileSystem fileSystem = FileSystem.get(URI.create("hdfs://node01:8020/outresult.txt"),configuration);
        //（2）创建输入流
        FSDataInputStream fsDataInputStream = fileSystem.open(new Path("hdfs://node01:8020/outresult.txt"));
        //（3）创建输出流
        FileOutputStream fileOutputStream = new FileOutputStream("E:\\hdfs\\aa.txt");
        //（4）流对拷
        IOUtils.copy(fsDataInputStream,fileOutputStream);
        //（5）关闭资源
        IOUtils.closeQuietly(fsDataInputStream);
        IOUtils.closeQuietly(fileOutputStream);
        fileSystem.close();
    }

    /**
     * 9、小文件合并
     */
    @Test
    public void mergeFile() throws URISyntaxException, IOException, InterruptedException {
        //获取分布式文件系统hdfs
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"),new Configuration(),"hadoop");
       
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("hdfs://node01:8020/bigfile.xml"));

        LocalFileSystem localFileSystem = FileSystem.getLocal(new Configuration());

        //获取本地文件系统 localFileSystem
        FileStatus[] fileStatuses = localFileSystem.listStatus(new Path("E:\\hdfs\\小文件合并"));
        //读取所有本地小文件,写入到hdfs的大文件里面去
        for(FileStatus fileStatus : fileStatuses){
            //获取每一个本地的文件路径
            Path path = fileStatus.getPath();
            //读取本地小文件
            FSDataInputStream fsDataInputStream = localFileSystem.open(path);
            IOUtils.copy(fsDataInputStream,fsDataOutputStream);
            IOUtils.closeQuietly(fsDataInputStream);

        }
        IOUtils.closeQuietly(fsDataOutputStream);
        localFileSystem.close();
        fileSystem.close();


    }

}