package com.zhengkw.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import static java.lang.System.out;
/**
* @ClassName:HDFSclient
* @author: zhengkw
* @description: HDFS客户端
* @date: 20/02/20上午 11:28
* @version:1.0
* @since: jdk 1.8
*/
public class HDFSclient {
private FileSystem fileSystem;
/**
* @descrption:创建目录 优先级
* @return: void
* @date: 20/02/20 上午 11:43
* @author: zhengkw
*/
@Test
public void testMkdirs() throws IOException, URISyntaxException, InterruptedException {
//获取文件系统
Configuration configuration = new Configuration();
//配置在集群上运行(core-site)
configuration.set("dfs.replication", "2");
//获取文件系统对象
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop103:9000"), configuration, "zhengkw");
//创建目录
boolean mkdirFlag = fileSystem.mkdirs(new Path("/user/zhengkw/sanguo"));
// 关闭资源
fileSystem.close();
}
/**
* @descrption:从本地上传
* @return: void
* @date: 20/02/20 下午 12:38
* @author: zhengkw
*/
@Test
public void testCopyFromLocalFile() throws URISyntaxException, IOException, InterruptedException {
//创建configuration对象
Configuration configuration = new Configuration();
//获取文件系统对象(url,config,username)
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop103:9000"), configuration, "zhengkw");
//上传
fileSystem.copyFromLocalFile(false, new Path("f:/123.txt"), new Path("/user/zhengkw/sanguo/345.txt"));
//关闭资源
fileSystem.close();
out.println("finish");
}
/**
* @descrption:初始化获取fs对象
* @return: void
* @date: 20/02/20 下午 2:28
* @author: zhengkw
*/
@Before
public void init() throws URISyntaxException, IOException, InterruptedException {
fileSystem = FileSystem.get(new URI("hdfs://hadoop103:9000"), new Configuration(), "zhengkw");
}
/***
* @descrption:下载
* @return: void
* @date: 20/02/20 下午 1:17
* @author: zhengkw
*/
@Test
public void testCopyToLocalFile() throws IOException {
fileSystem.copyToLocalFile(false, new Path("/user/zhengkw/sanguo/345.txt"), new Path("F:/test/mimiga.txt"), true);
out.println("finish!");
}
/**
* @descrption:关闭资源
* @return: void
* @date: 20/02/20 下午 2:28
* @author: zhengkw
*/
@After
public void close() throws IOException {
fileSystem.close();
}
/**
* @descrption:删除目录
* @return: void
* @date: 20/02/20 下午 2:31
* @author: zhengkw
*/
@Test
public void testDelete() throws IOException {
fileSystem.delete(new Path("/user"), true);//true 递归
}
/**
* @descrption: 修改文件名
* @return: void
* @date: 20/02/20 下午 2:37
* @author: zhengkw
*/
@Test
public void changeFileName() throws IOException {
fileSystem.rename(new Path("/user/zhengkw/sanguo/345.txt"), new Path("/user/zhengkw/sanguo/新目标.txt"));
}
/**
* @descrption:打印文件信息
* @return: void
* @date: 20/02/20 下午 3:15
* @author: zhengkw
*/
@Test
public void listFiles() throws IOException {
RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"), true);
while (listFiles.hasNext()) {
LocatedFileStatus status = listFiles.next();
out.println("这个是路径:" + status.getPath());
out.println("长度:" + status.getLen());
out.println("权限:" + status.getPermission());
out.println("块大小:" + status.getBlockSize());
out.println("分组:" + status.getGroup());
// 获取存储的块信息
BlockLocation[] blockLocations = status.getBlockLocations();
for (BlockLocation blockLocation : blockLocations
) {
// 获取块存储的主机节点
String[] hosts = blockLocation.getHosts();
for (String host : hosts
) {
out.println(host);
}
}
System.out.println("---------------分割线-----------------");
}
}
/**
* @descrption:判断文件夹还是文件
* @return: void
* @date: 20/02/20 下午 3:41
* @author: zhengkw
*/
@Test
public void judgeFileOrDirectory() throws IOException {
//1获取文件状态(当前目录)
FileStatus[] listStatus = fileSystem.listStatus(new Path("/user/zhengkw/sanguo"));
//遍历文件状态
for (FileStatus fs : listStatus
) {
// 2判断是文件还是文件夹
// 如果是文件输出这个是file,如果是目录输出这个是目录xxx
if (fs.isFile()) {
out.println("这个是文件:" + fs.getPath().getName());
} else {
out.println("这个是目录" + fs.getPath().getName());
}
}
}
}
Maven pom
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zhengkw</groupId>
<artifactId>HDFSclient</artifactId>
<version>1.0-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>RELEASE</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.8.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.2</version>
</dependency>
<!-- <dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.8</version>
<scope>system</scope>
<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
</dependency>-->
</dependencies>
</project>